ERF
Energy Research and Forecasting: An Atmospheric Modeling Code
ERF Class Reference

#include <ERF.H>

Inheritance diagram for ERF:
Collaboration diagram for ERF:

Public Member Functions

 ERF ()
 
 ~ERF () override
 
void ERF_shared ()
 
 ERF (ERF &&) noexcept=delete
 
ERFoperator= (ERF &&other) noexcept=delete
 
 ERF (const ERF &other)=delete
 
ERFoperator= (const ERF &other)=delete
 
void Evolve ()
 
void ErrorEst (int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override
 
void HurricaneTracker (int lev, const amrex::MultiFab &U_new, const amrex::MultiFab &V_new, const amrex::MultiFab &W_new, const amrex::Real velmag_threshold, const bool is_track_io, amrex::TagBoxArray *tags=nullptr)
 
std::string MakeVTKFilename (int nstep)
 
std::string MakeVTKFilename_TrackerCircle (int nstep)
 
std::string MakeVTKFilename_EyeTracker_xy (int nstep)
 
std::string MakeFilename_EyeTracker_latlon (int nstep)
 
std::string MakeFilename_EyeTracker_maxvel (int nstep)
 
void WriteVTKPolyline (const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
 
void WriteLinePlot (const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
 
void InitData ()
 
void InitData_pre ()
 
void InitData_post ()
 
void WriteMyEBSurface ()
 
void compute_divergence (int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::Geometry const &geom_at_lev)
 
void project_velocity (int lev, amrex::Real dt)
 
void project_momenta (int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars)
 
void project_velocity_tb (int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars)
 
void poisson_wall_dist (int lev)
 
void make_subdomains (const amrex::BoxList &ba, amrex::Vector< amrex::BoxArray > &bins)
 
void solve_with_EB_mlmg (int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
 
void solve_with_gmres (int lev, const amrex::Box &subdomain, amrex::MultiFab &rhs, amrex::MultiFab &p, amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &fluxes, amrex::MultiFab &ax_sub, amrex::MultiFab &ay_sub, amrex::MultiFab &az_sub, amrex::MultiFab &, amrex::MultiFab &znd_sub)
 
void solve_with_mlmg (int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
 
void ImposeBCsOnPhi (int lev, amrex::MultiFab &phi, const amrex::Box &subdomain)
 
amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > get_projection_bc (amrex::Orientation::Side side) const noexcept
 
bool projection_has_dirichlet (amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > bcs) const
 
void init_only (int lev, amrex::Real time)
 
void restart ()
 
void check_state_for_nans (amrex::MultiFab const &S)
 
void check_vels_for_nans (amrex::MultiFab const &xvel, amrex::MultiFab const &yvel, amrex::MultiFab const &zvel)
 
void check_for_negative_theta (amrex::MultiFab &S)
 
void check_for_low_temp (amrex::MultiFab &S)
 
bool writeNow (const amrex::Real cur_time, const int nstep, const int plot_int, const amrex::Real plot_per, const amrex::Real dt_0, amrex::Real &last_file_time)
 
void post_timestep (int nstep, amrex::Real time, amrex::Real dt_lev)
 
void sum_integrated_quantities (amrex::Real time)
 
void sum_derived_quantities (amrex::Real time)
 
void sum_energy_quantities (amrex::Real time)
 
void write_1D_profiles (amrex::Real time)
 
void write_1D_profiles_stag (amrex::Real time)
 
amrex::Real cloud_fraction (amrex::Real time)
 
void FillBdyCCVels (amrex::Vector< amrex::MultiFab > &mf_cc_vel, int levc=0)
 
void sample_points (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
 
void sample_lines (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
 
void derive_diag_profiles (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
 
void derive_diag_profiles_stag (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
 
void derive_stress_profiles (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
 
void derive_stress_profiles_stag (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
 
amrex::Real volWgtSumMF (int lev, const amrex::MultiFab &mf, int comp, bool finemask)
 
void MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
void RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
void ClearLevel (int lev) override
 
void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
amrex::Real estTimeStep (int lev, long &dt_fast_ratio) const
 
void advance_dycore (int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::MultiFab &buoyancy, amrex::Geometry fine_geom, amrex::Real dt, amrex::Real time)
 
void advance_microphysics (int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance, const int &iteration, const amrex::Real &time)
 
void advance_lsm (int lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, const amrex::Real &dt_advance)
 
void advance_radiation (int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance)
 
amrex::MultiFab & build_fine_mask (int lev)
 
void MakeHorizontalAverages ()
 
void MakeDiagnosticAverage (amrex::Vector< amrex::Real > &h_havg, amrex::MultiFab &S, int n)
 
void derive_upwp (amrex::Vector< amrex::Real > &h_havg)
 
void Write3DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
 
void Write2DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
 
void WriteSubvolume (amrex::Vector< std::string > subvol_var_names)
 
void WriteMultiLevelPlotfileWithTerrain (const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
 
void WriteGenericPlotfileHeaderWithTerrain (std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
 
void erf_enforce_hse (int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, amrex::MultiFab &qv, std::unique_ptr< amrex::MultiFab > &z_cc)
 
void init_from_input_sounding (int lev)
 
void init_immersed_forcing (int lev)
 
void input_sponge (int lev)
 
void init_from_hse (int lev)
 
void init_thin_body (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
 
void CreateWeatherDataGeomBoxArrayDistMap (const std::string &filename, amrex::Geometry &geom_weather, amrex::BoxArray &nba, amrex::DistributionMapping &dm)
 
void FillWeatherDataMultiFab (const std::string &filename, const amrex::Geometry &geom_weather, const amrex::BoxArray &nba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &weather_forecast_data)
 
void InterpWeatherDataOntoMesh (const amrex::Geometry &geom_weather, amrex::MultiFab &weather_forecast_interp, amrex::Vector< amrex::Vector< amrex::MultiFab >> &forecast_state)
 
void CreateForecastStateMultiFabs (amrex::Vector< amrex::Vector< amrex::MultiFab >> &forecast_state)
 
void WeatherDataInterpolation (const amrex::Real time)
 
void fill_from_bndryregs (const amrex::Vector< amrex::MultiFab * > &mfs, amrex::Real time)
 
void MakeEBGeometry ()
 
void make_eb_box ()
 
void make_eb_regular ()
 
void AverageDownTo (int crse_lev, int scomp, int ncomp)
 
void WriteCheckpointFile () const
 
void ReadCheckpointFile ()
 
void ReadCheckpointFileSurfaceLayer ()
 
void init_zphys (int lev, amrex::Real time)
 
void remake_zphys (int lev, amrex::Real time, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)
 
void update_terrain_arrays (int lev)
 
void writeJobInfo (const std::string &dir) const
 

Static Public Member Functions

static bool is_it_time_for_action (int nstep, amrex::Real time, amrex::Real dt, int action_interval, amrex::Real action_per)
 
static void writeBuildInfo (std::ostream &os)
 
static void print_banner (MPI_Comm, std::ostream &)
 
static void print_usage (MPI_Comm, std::ostream &)
 
static void print_error (MPI_Comm, const std::string &msg)
 
static void print_summary (std::ostream &)
 
static void print_tpls (std::ostream &)
 

Public Attributes

amrex::Vector< std::array< amrex::Real, 2 > > hurricane_track_xy
 
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_eye_track_xy
 
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_eye_track_latlon
 
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_maxvel_vs_time
 
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_tracker_circle
 
amrex::Vector< amrex::MultiFab > weather_forecast_data_1
 
amrex::Vector< amrex::MultiFab > weather_forecast_data_2
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_1
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_2
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_interp
 
std::string pp_prefix {"erf"}
 

Private Member Functions

void ReadParameters ()
 
void ParameterSanityChecks ()
 
void AverageDown ()
 
void update_diffusive_arrays (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
 
void Construct_ERFFillPatchers (int lev)
 
void Define_ERFFillPatchers (int lev)
 
void init1DArrays ()
 
void init_bcs ()
 
void init_custom (int lev)
 
void init_uniform (int lev)
 
void init_stuff (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)
 
void turbPert_update (const int lev, const amrex::Real dt)
 
void turbPert_amplitude (const int lev)
 
void initialize_integrator (int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)
 
void make_physbcs (int lev)
 
void initializeMicrophysics (const int &)
 
void FillPatchCrseLevel (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)
 
void FillPatchFineLevel (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, const amrex::MultiFab &old_base_state, const amrex::MultiFab &new_base_state, bool fillset=true, bool cons_only=false)
 
void FillIntermediatePatch (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons)
 
void FillCoarsePatch (int lev, amrex::Real time)
 
void timeStep (int lev, amrex::Real time, int iteration)
 
void Advance (int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle)
 
void initHSE ()
 Initialize HSE. More...
 
void initHSE (int lev)
 
void initRayleigh ()
 Initialize Rayleigh damping profiles. More...
 
void initSponge ()
 Initialize sponge profiles. More...
 
void setRayleighRefFromSounding (bool restarting)
 Set Rayleigh mean profiles from input sounding. More...
 
void setSpongeRefFromSounding (bool restarting)
 Set sponge mean profiles from input sounding. More...
 
void ComputeDt (int step=-1)
 
std::string PlotFileName (int lev) const
 
void setPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
 
void setPlotVariables2D (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
 
void appendPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
 
void setSubVolVariables (const std::string &pp_subvol_var_names, amrex::Vector< std::string > &subvol_var_names)
 
void init_Dirichlet_bc_data (const std::string input_file)
 
void InitializeFromFile ()
 
void InitializeLevelFromData (int lev, const amrex::MultiFab &initial_data)
 
void post_update (amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)
 
void fill_rhs (amrex::MultiFab &rhs_mf, const amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)
 
void init_geo_wind_profile (const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)
 
void refinement_criteria_setup ()
 
AMREX_FORCE_INLINE amrex::YAFluxRegister * getAdvFluxReg (int lev)
 
AMREX_FORCE_INLINE std::ostream & DataLog (int i)
 
AMREX_FORCE_INLINE std::ostream & DerDataLog (int i)
 
AMREX_FORCE_INLINE int NumDataLogs () noexcept
 
AMREX_FORCE_INLINE int NumDerDataLogs () noexcept
 
AMREX_FORCE_INLINE std::ostream & SamplePointLog (int i)
 
AMREX_FORCE_INLINE int NumSamplePointLogs () noexcept
 
AMREX_FORCE_INLINE std::ostream & SampleLineLog (int i)
 
AMREX_FORCE_INLINE int NumSampleLineLogs () noexcept
 
amrex::IntVect & SamplePoint (int i)
 
AMREX_FORCE_INLINE int NumSamplePoints () noexcept
 
amrex::IntVect & SampleLine (int i)
 
AMREX_FORCE_INLINE int NumSampleLines () noexcept
 
void setRecordDataInfo (int i, const std::string &filename)
 
void setRecordDerDataInfo (int i, const std::string &filename)
 
void setRecordEnergyDataInfo (int i, const std::string &filename)
 
void setRecordSamplePointInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
 
void setRecordSampleLineInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
 
std::string DataLogName (int i) const noexcept
 The filename of the ith datalog file. More...
 
std::string DerDataLogName (int i) const noexcept
 
std::string SamplePointLogName (int i) const noexcept
 The filename of the ith sampleptlog file. More...
 
std::string SampleLineLogName (int i) const noexcept
 The filename of the ith samplelinelog file. More...
 
eb_ const & get_eb (int lev) const noexcept
 
amrex::EBFArrayBoxFactory const & EBFactory (int lev) const noexcept
 

Static Private Member Functions

static amrex::Vector< std::string > PlotFileVarNames (amrex::Vector< std::string > plot_var_names)
 
static void GotoNextLine (std::istream &is)
 
static AMREX_FORCE_INLINE int ComputeGhostCells (const SolverChoice &sc)
 
static amrex::Real getCPUTime ()
 
static int nghost_eb_basic ()
 
static int nghost_eb_volume ()
 
static int nghost_eb_full ()
 

Private Attributes

amrex::Vector< std::unique_ptr< amrex::MultiFab > > lat_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > lon_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sinPhi_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > cosPhi_m
 
InputSoundingData input_sounding_data
 
InputSpongeData input_sponge_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > xvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > yvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > zvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > th_bc_data
 
std::unique_ptr< ProblemBaseprob = nullptr
 
amrex::Vector< int > num_boxes_at_level
 
amrex::Vector< int > num_files_at_level
 
amrex::Vector< amrex::Vector< amrex::Box > > boxes_at_level
 
amrex::Vector< int > istep
 
amrex::Vector< int > nsubsteps
 
amrex::Vector< amrex::Realt_new
 
amrex::Vector< amrex::Realt_old
 
amrex::Vector< amrex::Realdt
 
amrex::Vector< long > dt_mri_ratio
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_new
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_old
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > gradp
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vel_t_avg
 
amrex::Vector< amrex::Realt_avg_cnt
 
amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > > mri_integrator_mem
 
amrex::Vector< amrex::MultiFab > pp_inc
 
amrex::Vector< amrex::MultiFab > lagged_delta_rt
 
amrex::Vector< amrex::MultiFab > avg_xmom
 
amrex::Vector< amrex::MultiFab > avg_ymom
 
amrex::Vector< amrex::MultiFab > avg_zmom
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > > physbcs_cons
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > > physbcs_u
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > > physbcs_v
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > > physbcs_w
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > > physbcs_base
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Theta_prim
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qv_prim
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qr_prim
 
amrex::Vector< amrex::MultiFab > rU_old
 
amrex::Vector< amrex::MultiFab > rU_new
 
amrex::Vector< amrex::MultiFab > rV_old
 
amrex::Vector< amrex::MultiFab > rV_new
 
amrex::Vector< amrex::MultiFab > rW_old
 
amrex::Vector< amrex::MultiFab > rW_new
 
amrex::Vector< amrex::MultiFab > zmom_crse_rhs
 
std::unique_ptr< Microphysicsmicro
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > qmoist
 
LandSurface lsm
 
amrex::Vector< std::string > lsm_data_name
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_data
 
amrex::Vector< std::string > lsm_flux_name
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_flux
 
amrex::Vector< std::unique_ptr< IRadiation > > rad
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > qheating_rates
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > rad_fluxes
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sw_lw_fluxes
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > solar_zenith
 
bool plot_rad = false
 
int rad_datalog_int = -1
 
int cf_width {0}
 
int cf_set_width {0}
 
amrex::Vector< ERFFillPatcherFPr_c
 
amrex::Vector< ERFFillPatcherFPr_u
 
amrex::Vector< ERFFillPatcherFPr_v
 
amrex::Vector< ERFFillPatcherFPr_w
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau_corr
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > eddyDiffs_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SmnSmn_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > sst_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > tsk_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > lmask_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > land_type_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > soil_type_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > urb_frac_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx1_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx2_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx3_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_diss_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx1_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx2_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx3_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q2fx3_lev
 
amrex::Vector< amrex::Vector< amrex::Real > > zlevels_stag
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_t_rk
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > terrain_blanking
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > walldist
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > mapfac
 
amrex::Vector< amrex::Vector< amrex::Real > > stretched_dz_h
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > stretched_dz_d
 
amrex::Vector< amrex::MultiFab > base_state
 
amrex::Vector< amrex::MultiFab > base_state_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave_onegrid
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave_onegrid
 
bool finished_wave = false
 
amrex::Vector< amrex::YAFluxRegister * > advflux_reg
 
amrex::Vector< amrex::BCRec > domain_bcs_type
 
amrex::Gpu::DeviceVector< amrex::BCRec > domain_bcs_type_d
 
amrex::Array< std::string, 2 *AMREX_SPACEDIM > domain_bc_type
 
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_maxm_bc_extdir_vals
 
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_maxm_bc_neumann_vals
 
amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 > phys_bc_type
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > xflux_imask
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > yflux_imask
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > zflux_imask
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_xforce
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_yforce
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_zforce
 
const int datwidth = 14
 
const int datprecision = 6
 
const int timeprecision = 13
 
int max_step = -1
 
bool use_datetime = false
 
const std::string datetime_format = "%Y-%m-%d %H:%M:%S"
 
std::string restart_chkfile = ""
 
amrex::Vector< amrex::Realfixed_dt
 
amrex::Vector< amrex::Realfixed_fast_dt
 
int regrid_int = -1
 
bool regrid_level_0_on_restart = false
 
std::string plot3d_file_1 {"plt_1_"}
 
std::string plot3d_file_2 {"plt_2_"}
 
std::string plot2d_file_1 {"plt2d_1_"}
 
std::string plot2d_file_2 {"plt2d_2_"}
 
std::string subvol_file {"subvol"}
 
bool m_expand_plotvars_to_unif_rr = false
 
int m_plot3d_int_1 = -1
 
int m_plot3d_int_2 = -1
 
int m_plot2d_int_1 = -1
 
int m_plot2d_int_2 = -1
 
int m_subvol_int = -1
 
amrex::Real m_plot3d_per_1 = -1.0
 
amrex::Real m_plot3d_per_2 = -1.0
 
amrex::Real m_plot2d_per_1 = -1.0
 
amrex::Real m_plot2d_per_2 = -1.0
 
amrex::Real m_subvol_per = -1.0
 
bool m_plot_face_vels = false
 
bool plot_lsm = false
 
int profile_int = -1
 
bool destag_profiles = true
 
std::string check_file {"chk"}
 
int m_check_int = -1
 
amrex::Real m_check_per = -1.0
 
amrex::Vector< std::string > subvol3d_var_names
 
amrex::Vector< std::string > plot3d_var_names_1
 
amrex::Vector< std::string > plot3d_var_names_2
 
amrex::Vector< std::string > plot2d_var_names_1
 
amrex::Vector< std::string > plot2d_var_names_2
 
const amrex::Vector< std::string > cons_names
 
const amrex::Vector< std::string > derived_names
 
const amrex::Vector< std::string > derived_names_2d
 
const amrex::Vector< std::string > derived_subvol_names {"soundspeed", "temp", "theta", "KE", "scalar"}
 
TurbulentPerturbation turbPert
 
int real_width {0}
 
int real_set_width {0}
 
bool real_extrap_w {true}
 
bool metgrid_debug_quiescent {false}
 
bool metgrid_debug_isothermal {false}
 
bool metgrid_debug_dry {false}
 
bool metgrid_debug_psfc {false}
 
bool metgrid_debug_msf {false}
 
bool metgrid_interp_theta {false}
 
bool metgrid_basic_linear {false}
 
bool metgrid_use_below_sfc {true}
 
bool metgrid_use_sfc {true}
 
bool metgrid_retain_sfc {false}
 
amrex::Real metgrid_proximity {500.0}
 
int metgrid_order {2}
 
int metgrid_force_sfc_k {6}
 
amrex::Vector< amrex::BoxArray > ba1d
 
amrex::Vector< amrex::BoxArray > ba2d
 
std::unique_ptr< amrex::MultiFab > mf_C1H
 
std::unique_ptr< amrex::MultiFab > mf_C2H
 
std::unique_ptr< amrex::MultiFab > mf_MUB
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_PSFC
 
amrex::Vector< amrex::Vector< amrex::Real > > h_rhotheta_src
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhotheta_src
 
amrex::Vector< amrex::Vector< amrex::Real > > h_rhoqt_src
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhoqt_src
 
amrex::Vector< amrex::Vector< amrex::Real > > h_w_subsid
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_w_subsid
 
amrex::Vector< amrex::Vector< amrex::Real > > h_u_geos
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_u_geos
 
amrex::Vector< amrex::Vector< amrex::Real > > h_v_geos
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_v_geos
 
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_rayleigh_ptrs
 
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_sponge_ptrs
 
amrex::Vector< amrex::Vector< amrex::Real > > h_sinesq_ptrs
 
amrex::Vector< amrex::Vector< amrex::Real > > h_sinesq_stag_ptrs
 
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_rayleigh_ptrs
 
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_sponge_ptrs
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_sinesq_ptrs
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_sinesq_stag_ptrs
 
amrex::Vector< amrex::Realh_havg_density
 
amrex::Vector< amrex::Realh_havg_temperature
 
amrex::Vector< amrex::Realh_havg_pressure
 
amrex::Vector< amrex::Realh_havg_qv
 
amrex::Vector< amrex::Realh_havg_qc
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_density
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_temperature
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_pressure
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_qv
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_qc
 
std::unique_ptr< WriteBndryPlanesm_w2d = nullptr
 
std::unique_ptr< ReadBndryPlanesm_r2d = nullptr
 
std::unique_ptr< SurfaceLayerm_SurfaceLayer = nullptr
 
amrex::Vector< std::unique_ptr< ForestDrag > > m_forest_drag
 
amrex::Vector< amrex::Vector< amrex::BoxArray > > subdomains
 
amrex::MultiFab fine_mask
 
amrex::Vector< amrex::Realdz_min
 
int line_sampling_interval = -1
 
int plane_sampling_interval = -1
 
amrex::Real line_sampling_per = -1.0
 
amrex::Real plane_sampling_per = -1.0
 
std::unique_ptr< LineSamplerline_sampler = nullptr
 
std::unique_ptr< PlaneSamplerplane_sampler = nullptr
 
amrex::Vector< std::unique_ptr< std::fstream > > datalog
 
amrex::Vector< std::unique_ptr< std::fstream > > der_datalog
 
amrex::Vector< std::unique_ptr< std::fstream > > tot_e_datalog
 
amrex::Vector< std::string > datalogname
 
amrex::Vector< std::string > der_datalogname
 
amrex::Vector< std::string > tot_e_datalogname
 
amrex::Vector< std::unique_ptr< std::fstream > > sampleptlog
 
amrex::Vector< std::string > sampleptlogname
 
amrex::Vector< amrex::IntVect > samplepoint
 
amrex::Vector< std::unique_ptr< std::fstream > > samplelinelog
 
amrex::Vector< std::string > samplelinelogname
 
amrex::Vector< amrex::IntVect > sampleline
 
amrex::Vector< std::unique_ptr< eb_ > > eb
 

Static Private Attributes

static int last_plot3d_file_step_1 = -1
 
static int last_plot3d_file_step_2 = -1
 
static int last_plot2d_file_step_1 = -1
 
static int last_plot2d_file_step_2 = -1
 
static int last_check_file_step = -1
 
static int last_subvol_step = -1
 
static amrex::Real last_plot3d_file_time_1 = 0.0
 
static amrex::Real last_plot3d_file_time_2 = 0.0
 
static amrex::Real last_plot2d_file_time_1 = 0.0
 
static amrex::Real last_plot2d_file_time_2 = 0.0
 
static amrex::Real last_check_file_time = 0.0
 
static amrex::Real last_subvol_time = 0.0
 
static bool plot_file_on_restart = true
 
static amrex::Real start_time = 0.0
 
static amrex::Real stop_time = std::numeric_limits<amrex::Real>::max()
 
static amrex::Real cfl = 0.8
 
static amrex::Real sub_cfl = 1.0
 
static amrex::Real init_shrink = 1.0
 
static amrex::Real change_max = 1.1
 
static amrex::Real dt_max_initial = 2.0e100
 
static amrex::Real dt_max = 1.0e9
 
static int fixed_mri_dt_ratio = 0
 
static SolverChoice solverChoice
 
static int verbose = 0
 
static int mg_verbose = 0
 
static bool use_fft = false
 
static int check_for_nans = 0
 
static int sum_interval = -1
 
static int pert_interval = -1
 
static amrex::Real sum_per = -1.0
 
static PlotFileType plotfile3d_type_1 = PlotFileType::None
 
static PlotFileType plotfile3d_type_2 = PlotFileType::None
 
static PlotFileType plotfile2d_type_1 = PlotFileType::None
 
static PlotFileType plotfile2d_type_2 = PlotFileType::None
 
static StateInterpType interpolation_type
 
static std::string sponge_type
 
static amrex::Vector< amrex::Vector< std::string > > nc_init_file = {{""}}
 
static std::string nc_bdy_file
 
static std::string nc_low_file
 
static int output_1d_column = 0
 
static int column_interval = -1
 
static amrex::Real column_per = -1.0
 
static amrex::Real column_loc_x = 0.0
 
static amrex::Real column_loc_y = 0.0
 
static std::string column_file_name = "column_data.nc"
 
static int output_bndry_planes = 0
 
static int bndry_output_planes_interval = -1
 
static amrex::Real bndry_output_planes_per = -1.0
 
static amrex::Real bndry_output_planes_start_time = 0.0
 
static int input_bndry_planes = 0
 
static int ng_dens_hse
 
static int ng_pres_hse
 
static amrex::Vector< amrex::AMRErrorTag > ref_tags
 
static amrex::Real startCPUTime = 0.0
 
static amrex::Real previousCPUTimeUsed = 0.0
 

Detailed Description

Main class in ERF code, instantiated from main.cpp

Constructor & Destructor Documentation

◆ ERF() [1/3]

ERF::ERF ( )
129 {
130  int fix_random_seed = 0;
131  ParmParse pp("erf"); pp.query("fix_random_seed", fix_random_seed);
132  // Note that the value of 1024UL is not significant -- the point here is just to set the
133  // same seed for all MPI processes for the purpose of regression testing
134  if (fix_random_seed) {
135  Print() << "Fixing the random seed" << std::endl;
136  InitRandom(1024UL);
137  }
138 
139  ERF_shared();
140 }
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real pp(amrex::Real y)
Definition: ERF_MicrophysicsUtils.H:233
void ERF_shared()
Definition: ERF.cpp:143
Here is the call graph for this function:

◆ ~ERF()

ERF::~ERF ( )
overridedefault

◆ ERF() [2/3]

ERF::ERF ( ERF &&  )
deletenoexcept

◆ ERF() [3/3]

ERF::ERF ( const ERF other)
delete

Member Function Documentation

◆ Advance()

void ERF::Advance ( int  lev,
amrex::Real  time,
amrex::Real  dt_lev,
int  iteration,
int  ncycle 
)
private

Function that advances the solution at one level for a single time step – this does some preliminaries then calls erf_advance

Parameters
[in]levlevel of refinement (coarsest level is 0)
[in]timestart time for time advance
[in]dt_levtime step for this time advance
21 {
22  BL_PROFILE("ERF::Advance()");
23 
24  // We must swap the pointers so the previous step's "new" is now this step's "old"
25  std::swap(vars_old[lev], vars_new[lev]);
26 
27  MultiFab& S_old = vars_old[lev][Vars::cons];
28  MultiFab& S_new = vars_new[lev][Vars::cons];
29 
30  MultiFab& U_old = vars_old[lev][Vars::xvel];
31  MultiFab& V_old = vars_old[lev][Vars::yvel];
32  MultiFab& W_old = vars_old[lev][Vars::zvel];
33 
34  MultiFab& U_new = vars_new[lev][Vars::xvel];
35  MultiFab& V_new = vars_new[lev][Vars::yvel];
36  MultiFab& W_new = vars_new[lev][Vars::zvel];
37 
38  // We need to set these because otherwise in the first call to erf_advance we may
39  // read uninitialized data on ghost values in setting the bc's on the velocities
40  U_new.setVal(1.e34,U_new.nGrowVect());
41  V_new.setVal(1.e34,V_new.nGrowVect());
42  W_new.setVal(1.e34,W_new.nGrowVect());
43 
44  // Do error checking for negative (rho theta) here
45  if (solverChoice.anelastic[lev] != 1) {
47  }
48 
49  //
50  // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
51  // If lev == 0 we have already FillPatched this in ERF::TimeStep
52  //
53  if (lev > 0) {
54  FillPatchFineLevel(lev, time, {&S_old, &U_old, &V_old, &W_old},
55  {&S_old, &rU_old[lev], &rV_old[lev], &rW_old[lev]},
56  base_state[lev], base_state[lev]);
57  }
58 
59  //
60  // So we must convert the fillpatched to momenta, including the ghost values
61  //
62  VelocityToMomentum(U_old, rU_old[lev].nGrowVect(),
63  V_old, rV_old[lev].nGrowVect(),
64  W_old, rW_old[lev].nGrowVect(),
65  S_old, rU_old[lev], rV_old[lev], rW_old[lev],
66  Geom(lev).Domain(),
68 
69  // Update the inflow perturbation update time and amplitude
70  if (solverChoice.pert_type == PerturbationType::Source ||
71  solverChoice.pert_type == PerturbationType::Direct ||
72  solverChoice.pert_type == PerturbationType::CPM)
73  {
74  turbPert.calc_tpi_update(lev, dt_lev, U_old, V_old, S_old);
75  }
76 
77  // If PerturbationType::Direct or CPM is selected, directly add the computed perturbation
78  // on the conserved field
79  if (solverChoice.pert_type == PerturbationType::Direct ||
80  solverChoice.pert_type == PerturbationType::CPM)
81  {
82  auto m_ixtype = S_old.boxArray().ixType(); // Conserved term
83  for (MFIter mfi(S_old,TileNoZ()); mfi.isValid(); ++mfi) {
84  Box bx = mfi.tilebox();
85  const Array4<Real> &cell_data = S_old.array(mfi);
86  const Array4<const Real> &pert_cell = turbPert.pb_cell[lev].array(mfi);
87  turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cell_data, pert_cell);
88  }
89  }
90 
91  // configure SurfaceLayer params if needed
92  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
93  if (m_SurfaceLayer) {
94  IntVect ng = Theta_prim[lev]->nGrowVect();
95  MultiFab::Copy( *Theta_prim[lev], S_old, RhoTheta_comp, 0, 1, ng);
96  MultiFab::Divide(*Theta_prim[lev], S_old, Rho_comp , 0, 1, ng);
97  if (solverChoice.moisture_type != MoistureType::None) {
98  ng = Qv_prim[lev]->nGrowVect();
99 
100  MultiFab::Copy( *Qv_prim[lev], S_old, RhoQ1_comp, 0, 1, ng);
101  MultiFab::Divide(*Qv_prim[lev], S_old, Rho_comp , 0, 1, ng);
102 
103  if (solverChoice.moisture_indices.qr > -1) {
104  MultiFab::Copy( *Qr_prim[lev], S_old, solverChoice.moisture_indices.qr, 0, 1, ng);
105  MultiFab::Divide(*Qr_prim[lev], S_old, Rho_comp , 0, 1, ng);
106  } else {
107  Qr_prim[lev]->setVal(0.0);
108  }
109  }
110  // NOTE: std::swap above causes the field ptrs to be out of date.
111  // Reassign the field ptrs for MAC avg computation.
112  m_SurfaceLayer->update_mac_ptrs(lev, vars_old, Theta_prim, Qv_prim, Qr_prim);
113  m_SurfaceLayer->update_pblh(lev, vars_old, z_phys_cc[lev].get(),
115  m_SurfaceLayer->update_fluxes(lev, time, S_old, z_phys_nd[lev]);
116  }
117  }
118 
119 #if defined(ERF_USE_WINDFARM)
120  // **************************************************************************************
121  // Update the windfarm sources
122  // **************************************************************************************
123  if (solverChoice.windfarm_type != WindFarmType::None) {
124  advance_windfarm(Geom(lev), dt_lev, S_old,
125  U_old, V_old, W_old, vars_windfarm[lev],
126  Nturb[lev], SMark[lev], time);
127  }
128 
129 #endif
130 
131  // **************************************************************************************
132  // Weather data interpolation
133  // **************************************************************************************
134  if(solverChoice.init_type == InitType::HindCast and
137  }
138 
139  // **************************************************************************************
140  // Update the radiation sources with the "old" state
141  // **************************************************************************************
142  advance_radiation(lev, S_old, dt_lev);
143 
144 #ifdef ERF_USE_SHOC
145  // **************************************************************************************
146  // Update the "old" state using SHOC
147  // **************************************************************************************
148  if (solverChoice.use_shoc) {
149  // Get SFC fluxes from SurfaceLayer
150  if (m_SurfaceLayer) {
151  Vector<const MultiFab*> mfs = {&S_old, &U_old, &V_old, &W_old};
152  m_SurfaceLayer->impose_SurfaceLayer_bcs(lev, mfs, Tau[lev],
153  SFS_hfx1_lev[lev].get() , SFS_hfx2_lev[lev].get() , SFS_hfx3_lev[lev].get(),
154  SFS_q1fx1_lev[lev].get(), SFS_q1fx2_lev[lev].get(), SFS_q1fx3_lev[lev].get(),
155  z_phys_nd[lev].get());
156  }
157 
158  // Get Shoc tendencies and update the state
159  Real* w_sub = (solverChoice.custom_w_subsidence) ? d_w_subsid[lev].data() : nullptr;
160  compute_shoc_tendencies(lev, &S_old, &U_old, &V_old, &W_old, w_sub,
161  Tau[lev][TauType::tau13].get(), Tau[lev][TauType::tau23].get(),
162  SFS_hfx3_lev[lev].get() , SFS_q1fx3_lev[lev].get() ,
163  eddyDiffs_lev[lev].get() , z_phys_nd[lev].get() ,
164  dt_lev);
165  }
166 #endif
167 
168  const BoxArray& ba = S_old.boxArray();
169  const DistributionMapping& dm = S_old.DistributionMap();
170 
171  int nvars = S_old.nComp();
172 
173  // Source array for conserved cell-centered quantities -- this will be filled
174  // in the call to make_sources in ERF_TI_slow_rhs_pre.H
175  MultiFab cc_source(ba,dm,nvars,1); cc_source.setVal(0.0);
176 
177  // Source arrays for momenta -- these will be filled
178  // in the call to make_mom_sources in ERF_TI_slow_rhs_pre.H
179  BoxArray ba_x(ba); ba_x.surroundingNodes(0);
180  MultiFab xmom_source(ba_x,dm,1,1); xmom_source.setVal(0.0);
181 
182  BoxArray ba_y(ba); ba_y.surroundingNodes(1);
183  MultiFab ymom_source(ba_y,dm,1,1); ymom_source.setVal(0.0);
184 
185  BoxArray ba_z(ba); ba_z.surroundingNodes(2);
186  MultiFab zmom_source(ba_z,dm,1,1); zmom_source.setVal(0.0);
187  MultiFab buoyancy(ba_z,dm,1,1); buoyancy.setVal(0.0);
188 
189  amrex::Vector<MultiFab> state_old;
190  amrex::Vector<MultiFab> state_new;
191 
192  // **************************************************************************************
193  // Here we define state_old and state_new which are to be advanced
194  // **************************************************************************************
195  // Initial solution
196  // Note that "old" and "new" here are relative to each RK stage.
197  state_old.push_back(MultiFab(S_old , amrex::make_alias, 0, nvars)); // cons
198  state_old.push_back(MultiFab(rU_old[lev], amrex::make_alias, 0, 1)); // xmom
199  state_old.push_back(MultiFab(rV_old[lev], amrex::make_alias, 0, 1)); // ymom
200  state_old.push_back(MultiFab(rW_old[lev], amrex::make_alias, 0, 1)); // zmom
201 
202  // Final solution
203  // state_new at the end of the last RK stage holds the t^{n+1} data
204  state_new.push_back(MultiFab(S_new , amrex::make_alias, 0, nvars)); // cons
205  state_new.push_back(MultiFab(rU_new[lev], amrex::make_alias, 0, 1)); // xmom
206  state_new.push_back(MultiFab(rV_new[lev], amrex::make_alias, 0, 1)); // ymom
207  state_new.push_back(MultiFab(rW_new[lev], amrex::make_alias, 0, 1)); // zmom
208 
209  // **************************************************************************************
210  // Tests on the reasonableness of the solution
211  // **************************************************************************************
212  // Test for NaNs after dycore
213  if (check_for_nans > 1) {
214  amrex::Print() << "Testing old state and vels for NaNs before dycore" << std::endl;
215  check_state_for_nans(S_old);
216  check_vels_for_nans(rU_old[lev],rV_old[lev],rW_old[lev]);
217  }
218 
219  // **************************************************************************************
220  // Update the dycore
221  // **************************************************************************************
222  advance_dycore(lev, state_old, state_new,
223  U_old, V_old, W_old,
224  U_new, V_new, W_new,
225  cc_source, xmom_source, ymom_source, zmom_source, buoyancy,
226  Geom(lev), dt_lev, time);
227 
228  // **************************************************************************************
229  // Tests on the reasonableness of the solution
230  // **************************************************************************************
231  // Test for NaNs after dycore
232  if (check_for_nans > 0) {
233  amrex::Print() << "Testing new state and vels for NaNs after dycore" << std::endl;
234  check_state_for_nans(S_new);
235  check_vels_for_nans(rU_new[lev],rV_new[lev],rW_new[lev]);
236  }
237 
238  // We only test on low temp if we have a moisture model because we are protecting against
239  // the test on low temp inside the moisture models
240  if (solverChoice.anelastic[lev] != 1) {
241  if (solverChoice.moisture_type != MoistureType::None) {
242  check_for_low_temp(S_new);
243  }
244  else
245  {
246  // Otherwise we will test on negative (rhotheta) coming out of the dycore
248  }
249  }
250 
251  // **************************************************************************************
252  // Update the microphysics (moisture)
253  // **************************************************************************************
255  {
256  advance_microphysics(lev, S_new, dt_lev, iteration, time);
257 
258  // Test for NaNs after microphysics
259  if (check_for_nans > 0) {
260  amrex::Print() << "Testing new state for NaNs after advance_microphysics" << std::endl;
261  check_state_for_nans(S_new);
262  }
263  }
264 
265  // **************************************************************************************
266  // Update the land surface model
267  // **************************************************************************************
268  advance_lsm(lev, S_new, U_new, V_new, dt_lev);
269 
270 #ifdef ERF_USE_PARTICLES
271  // **************************************************************************************
272  // Update the particle positions
273  // **************************************************************************************
274  evolveTracers( lev, dt_lev, vars_new, z_phys_nd );
275 #endif
276 
277  // ***********************************************************************************************
278  // Impose domain boundary conditions here so that in FillPatching the fine data we won't
279  // need to re-fill these
280  // ***********************************************************************************************
281  if (lev < finest_level) {
282  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
284  0,vars_new[lev][Vars::cons].nComp(),
285  vars_new[lev][Vars::cons].nGrowVect(),time,BCVars::cons_bc,true);
286  (*physbcs_u[lev])(vars_new[lev][Vars::xvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
287  ngvect_vels,time,BCVars::xvel_bc,true);
288  (*physbcs_v[lev])(vars_new[lev][Vars::yvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
289  ngvect_vels,time,BCVars::yvel_bc,true);
290  (*physbcs_w[lev])(vars_new[lev][Vars::zvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
291  ngvect_vels,time,BCVars::zvel_bc,true);
292  }
293 
294  // **************************************************************************************
295  // Register old and new coarse data if we are at a level less than the finest level
296  // **************************************************************************************
297  if (lev < finest_level) {
298  if (cf_width > 0) {
299  // We must fill the ghost cells of these so that the parallel copy works correctly
300  state_old[IntVars::cons].FillBoundary(geom[lev].periodicity());
301  state_new[IntVars::cons].FillBoundary(geom[lev].periodicity());
302  FPr_c[lev].RegisterCoarseData({&state_old[IntVars::cons], &state_new[IntVars::cons]},
303  {time, time+dt_lev});
304  }
305 
306  if (cf_width >= 0) {
307  // We must fill the ghost cells of these so that the parallel copy works correctly
308  state_old[IntVars::xmom].FillBoundary(geom[lev].periodicity());
309  state_new[IntVars::xmom].FillBoundary(geom[lev].periodicity());
310  FPr_u[lev].RegisterCoarseData({&state_old[IntVars::xmom], &state_new[IntVars::xmom]},
311  {time, time+dt_lev});
312 
313  state_old[IntVars::ymom].FillBoundary(geom[lev].periodicity());
314  state_new[IntVars::ymom].FillBoundary(geom[lev].periodicity());
315  FPr_v[lev].RegisterCoarseData({&state_old[IntVars::ymom], &state_new[IntVars::ymom]},
316  {time, time+dt_lev});
317 
318  state_old[IntVars::zmom].FillBoundary(geom[lev].periodicity());
319  state_new[IntVars::zmom].FillBoundary(geom[lev].periodicity());
320  FPr_w[lev].RegisterCoarseData({&state_old[IntVars::zmom], &state_new[IntVars::zmom]},
321  {time, time+dt_lev});
322  }
323 
324  //
325  // Now create a MultiFab that holds (S_new - S_old) / dt from the coarse level interpolated
326  // on to the coarse/fine boundary at the fine resolution
327  //
328  Interpolater* mapper_f = &face_cons_linear_interp;
329 
330  // PhysBCFunctNoOp null_bc;
331  // MultiFab tempx(vars_new[lev+1][Vars::xvel].boxArray(),vars_new[lev+1][Vars::xvel].DistributionMap(),1,0);
332  // tempx.setVal(0.0);
333  // xmom_crse_rhs[lev+1].setVal(0.0);
334  // FPr_u[lev].FillSet(tempx , time , null_bc, domain_bcs_type);
335  // FPr_u[lev].FillSet(xmom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
336  // MultiFab::Subtract(xmom_crse_rhs[lev+1],tempx,0,0,1,IntVect{0});
337  // xmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
338 
339  // MultiFab tempy(vars_new[lev+1][Vars::yvel].boxArray(),vars_new[lev+1][Vars::yvel].DistributionMap(),1,0);
340  // tempy.setVal(0.0);
341  // ymom_crse_rhs[lev+1].setVal(0.0);
342  // FPr_v[lev].FillSet(tempy , time , null_bc, domain_bcs_type);
343  // FPr_v[lev].FillSet(ymom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
344  // MultiFab::Subtract(ymom_crse_rhs[lev+1],tempy,0,0,1,IntVect{0});
345  // ymom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
346 
347  MultiFab temp_state(zmom_crse_rhs[lev+1].boxArray(),zmom_crse_rhs[lev+1].DistributionMap(),1,0);
348  InterpFromCoarseLevel(temp_state, IntVect{0}, IntVect{0}, state_old[IntVars::zmom], 0, 0, 1,
349  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
350  InterpFromCoarseLevel(zmom_crse_rhs[lev+1], IntVect{0}, IntVect{0}, state_new[IntVars::zmom], 0, 0, 1,
351  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
352  MultiFab::Subtract(zmom_crse_rhs[lev+1],temp_state,0,0,1,IntVect{0});
353  zmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
354  }
355 
356  // ***********************************************************************************************
357  // Update the time averaged velocities if they are requested
358  // ***********************************************************************************************
360  Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(), U_new, V_new, W_new);
361  }
362 }
@ tau23
Definition: ERF_DataStruct.H:31
@ tau13
Definition: ERF_DataStruct.H:31
@ nvars
Definition: ERF_DataStruct.H:92
#define Rho_comp
Definition: ERF_IndexDefines.H:36
#define RhoTheta_comp
Definition: ERF_IndexDefines.H:37
#define RhoQ1_comp
Definition: ERF_IndexDefines.H:42
@ surface_layer
amrex::Real Real
Definition: ERF_ShocInterface.H:19
AMREX_FORCE_INLINE amrex::IntVect TileNoZ()
Definition: ERF_TileNoZ.H:11
void Time_Avg_Vel_atCC(const Real &dt, Real &t_avg_cnt, MultiFab *vel_t_avg, MultiFab &xvel, MultiFab &yvel, MultiFab &zvel)
Definition: ERF_TimeAvgVel.cpp:9
void VelocityToMomentum(const amrex::MultiFab &xvel_in, const amrex::IntVect &xvel_ngrow, const amrex::MultiFab &yvel_in, const amrex::IntVect &yvel_ngrow, const amrex::MultiFab &zvel_in, const amrex::IntVect &zvel_ngrow, const amrex::MultiFab &cons_in, amrex::MultiFab &xmom_out, amrex::MultiFab &ymom_out, amrex::MultiFab &zmom_out, const amrex::Box &domain, const amrex::Vector< amrex::BCRec > &domain_bcs_type_h, const amrex::MultiFab *c_vfrac=nullptr)
amrex::Vector< amrex::MultiFab > rU_new
Definition: ERF.H:846
void check_vels_for_nans(amrex::MultiFab const &xvel, amrex::MultiFab const &yvel, amrex::MultiFab const &zvel)
Definition: ERF.cpp:2810
amrex::Vector< ERFFillPatcher > FPr_u
Definition: ERF.H:900
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx3_lev
Definition: ERF.H:923
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_new
Definition: ERF.H:811
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx3_lev
Definition: ERF.H:921
amrex::Vector< ERFFillPatcher > FPr_v
Definition: ERF.H:901
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx1_lev
Definition: ERF.H:921
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > > physbcs_cons
Definition: ERF.H:833
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc
Definition: ERF.H:931
amrex::Vector< std::unique_ptr< amrex::MultiFab > > eddyDiffs_lev
Definition: ERF.H:907
static SolverChoice solverChoice
Definition: ERF.H:1160
amrex::Vector< ERFFillPatcher > FPr_c
Definition: ERF.H:899
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau
Definition: ERF.H:905
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vel_t_avg
Definition: ERF.H:818
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > > physbcs_w
Definition: ERF.H:836
amrex::Vector< amrex::MultiFab > base_state
Definition: ERF.H:962
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qv_prim
Definition: ERF.H:841
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx2_lev
Definition: ERF.H:923
amrex::Vector< amrex::MultiFab > rV_new
Definition: ERF.H:848
amrex::Vector< amrex::BCRec > domain_bcs_type
Definition: ERF.H:978
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qr_prim
Definition: ERF.H:842
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > > physbcs_u
Definition: ERF.H:834
amrex::Vector< amrex::Real > t_avg_cnt
Definition: ERF.H:819
void FillPatchFineLevel(int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, const amrex::MultiFab &old_base_state, const amrex::MultiFab &new_base_state, bool fillset=true, bool cons_only=false)
Definition: ERF_FillPatch.cpp:20
amrex::Vector< amrex::MultiFab > rU_old
Definition: ERF.H:845
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Theta_prim
Definition: ERF.H:840
static int check_for_nans
Definition: ERF.H:1199
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > > physbcs_v
Definition: ERF.H:835
void check_state_for_nans(amrex::MultiFab const &S)
Definition: ERF.cpp:2787
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd
Definition: ERF.H:930
void advance_dycore(int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::MultiFab &buoyancy, amrex::Geometry fine_geom, amrex::Real dt, amrex::Real time)
Definition: ERF_AdvanceDycore.cpp:38
amrex::Vector< amrex::MultiFab > rW_new
Definition: ERF.H:850
amrex::Vector< amrex::MultiFab > zmom_crse_rhs
Definition: ERF.H:854
void check_for_low_temp(amrex::MultiFab &S)
Definition: ERF.cpp:2837
void advance_lsm(int lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, const amrex::Real &dt_advance)
Definition: ERF_AdvanceLSM.cpp:5
TurbulentPerturbation turbPert
Definition: ERF.H:1163
amrex::Vector< amrex::MultiFab > rW_old
Definition: ERF.H:849
void check_for_negative_theta(amrex::MultiFab &S)
Definition: ERF.cpp:2872
std::unique_ptr< SurfaceLayer > m_SurfaceLayer
Definition: ERF.H:1329
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_w_subsid
Definition: ERF.H:1280
amrex::Vector< ERFFillPatcher > FPr_w
Definition: ERF.H:902
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx2_lev
Definition: ERF.H:921
amrex::Vector< amrex::Real > dt
Definition: ERF.H:805
void advance_radiation(int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance)
Definition: ERF_AdvanceRadiation.cpp:5
void advance_microphysics(int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance, const int &iteration, const amrex::Real &time)
Definition: ERF_AdvanceMicrophysics.cpp:5
int cf_width
Definition: ERF.H:897
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx1_lev
Definition: ERF.H:923
void WeatherDataInterpolation(const amrex::Real time)
Definition: ERF_WeatherDataInterpolation.cpp:506
amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 > phys_bc_type
Definition: ERF.H:991
amrex::Vector< amrex::MultiFab > rV_old
Definition: ERF.H:847
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_old
Definition: ERF.H:812
@ zvel_bc
Definition: ERF_IndexDefines.H:89
@ yvel_bc
Definition: ERF_IndexDefines.H:88
@ cons_bc
Definition: ERF_IndexDefines.H:76
@ xvel_bc
Definition: ERF_IndexDefines.H:87
@ ymom
Definition: ERF_IndexDefines.H:160
@ cons
Definition: ERF_IndexDefines.H:158
@ zmom
Definition: ERF_IndexDefines.H:161
@ xmom
Definition: ERF_IndexDefines.H:159
@ ng
Definition: ERF_Morrison.H:48
@ xvel
Definition: ERF_IndexDefines.H:141
@ cons
Definition: ERF_IndexDefines.H:140
@ zvel
Definition: ERF_IndexDefines.H:143
@ yvel
Definition: ERF_IndexDefines.H:142
int qr
Definition: ERF_DataStruct.H:104
bool use_shoc
Definition: ERF_DataStruct.H:1011
bool hindcast_lateral_forcing
Definition: ERF_DataStruct.H:1071
bool moisture_tight_coupling
Definition: ERF_DataStruct.H:1048
bool custom_w_subsidence
Definition: ERF_DataStruct.H:999
amrex::Vector< int > anelastic
Definition: ERF_DataStruct.H:927
MoistureType moisture_type
Definition: ERF_DataStruct.H:1027
PerturbationType pert_type
Definition: ERF_DataStruct.H:1017
WindFarmType windfarm_type
Definition: ERF_DataStruct.H:1028
static InitType init_type
Definition: ERF_DataStruct.H:895
MoistureComponentIndices moisture_indices
Definition: ERF_DataStruct.H:1046
bool time_avg_vel
Definition: ERF_DataStruct.H:1014
amrex::Vector< amrex::MultiFab > pb_cell
Definition: ERF_TurbPertStruct.H:640
void calc_tpi_update(const int lev, const amrex::Real dt, amrex::MultiFab &mf_xvel, amrex::MultiFab &mf_yvel, amrex::MultiFab &mf_cons)
Definition: ERF_TurbPertStruct.H:223
void apply_tpi(const int &lev, const amrex::Box &vbx, const int &comp, const amrex::IndexType &m_ixtype, const amrex::Array4< amrex::Real > &src_arr, const amrex::Array4< amrex::Real const > &pert_cell)
Definition: ERF_TurbPertStruct.H:324
Here is the call graph for this function:

◆ advance_dycore()

void ERF::advance_dycore ( int  level,
amrex::Vector< amrex::MultiFab > &  state_old,
amrex::Vector< amrex::MultiFab > &  state_new,
amrex::MultiFab &  xvel_old,
amrex::MultiFab &  yvel_old,
amrex::MultiFab &  zvel_old,
amrex::MultiFab &  xvel_new,
amrex::MultiFab &  yvel_new,
amrex::MultiFab &  zvel_new,
amrex::MultiFab &  source,
amrex::MultiFab &  xmom_src,
amrex::MultiFab &  ymom_src,
amrex::MultiFab &  zmom_src,
amrex::MultiFab &  buoyancy,
amrex::Geometry  fine_geom,
amrex::Real  dt,
amrex::Real  time 
)

Function that advances the solution at one level for a single time step – this sets up the multirate time integrator and calls the integrator's advance function

Parameters
[in]levellevel of refinement (coarsest level is 0)
[in]state_oldold-time conserved variables
[in]state_newnew-time conserved variables
[in]xvel_oldold-time x-component of velocity
[in]yvel_oldold-time y-component of velocity
[in]zvel_oldold-time z-component of velocity
[in]xvel_newnew-time x-component of velocity
[in]yvel_newnew-time y-component of velocity
[in]zvel_newnew-time z-component of velocity
[in]cc_srcsource term for conserved variables
[in]xmom_srcsource term for x-momenta
[in]ymom_srcsource term for y-momenta
[in]zmom_srcsource term for z-momenta
[in]fine_geomcontainer for geometry information at current level
[in]dt_advancetime step for this time advance
[in]old_timeold time for this time advance
48 {
49  BL_PROFILE_VAR("erf_advance_dycore()",erf_advance_dycore);
50 
51  const Box& domain = fine_geom.Domain();
52 
56 
57  MultiFab r_hse (base_state[level], make_alias, BaseState::r0_comp , 1);
58  MultiFab p_hse (base_state[level], make_alias, BaseState::p0_comp , 1);
59  MultiFab pi_hse(base_state[level], make_alias, BaseState::pi0_comp, 1);
60 
61  // These pointers are used in the MRI utility functions
62  MultiFab* r0 = &r_hse;
63  MultiFab* p0 = &p_hse;
64  MultiFab* pi0 = &pi_hse;
65 
66  Real* dptr_rhotheta_src = solverChoice.custom_rhotheta_forcing ? d_rhotheta_src[level].data() : nullptr;
67  Real* dptr_rhoqt_src = solverChoice.custom_moisture_forcing ? d_rhoqt_src[level].data() : nullptr;
68  Real* dptr_wbar_sub = solverChoice.custom_w_subsidence ? d_w_subsid[level].data() : nullptr;
69 
70  // Turbulent Perturbation Pointer
71  //Real* dptr_rhotheta_src = solverChoice.pert_type ? d_rhotheta_src[level].data() : nullptr;
72 
73  Vector<Real*> d_rayleigh_ptrs_at_lev;
74  d_rayleigh_ptrs_at_lev.resize(Rayleigh::nvars);
75  d_rayleigh_ptrs_at_lev[Rayleigh::ubar] = solverChoice.dampingChoice.rayleigh_damp_U ? d_rayleigh_ptrs[level][Rayleigh::ubar].data() : nullptr;
76  d_rayleigh_ptrs_at_lev[Rayleigh::vbar] = solverChoice.dampingChoice.rayleigh_damp_V ? d_rayleigh_ptrs[level][Rayleigh::vbar].data() : nullptr;
77  d_rayleigh_ptrs_at_lev[Rayleigh::wbar] = solverChoice.dampingChoice.rayleigh_damp_W ? d_rayleigh_ptrs[level][Rayleigh::wbar].data() : nullptr;
78  d_rayleigh_ptrs_at_lev[Rayleigh::thetabar] = solverChoice.dampingChoice.rayleigh_damp_T ? d_rayleigh_ptrs[level][Rayleigh::thetabar].data() : nullptr;
79 
80  bool use_rayleigh =
83  Real* d_sinesq_at_lev = (use_rayleigh) ? d_sinesq_ptrs[level].data() : nullptr;
84  Real* d_sinesq_stag_at_lev = (use_rayleigh) ? d_sinesq_stag_ptrs[level].data() : nullptr;
85 
86  Vector<Real*> d_sponge_ptrs_at_lev;
87  if(sc.sponge_type=="input_sponge")
88  {
89  d_sponge_ptrs_at_lev.resize(Sponge::nvars_sponge);
90  d_sponge_ptrs_at_lev[Sponge::ubar_sponge] = d_sponge_ptrs[level][Sponge::ubar_sponge].data();
91  d_sponge_ptrs_at_lev[Sponge::vbar_sponge] = d_sponge_ptrs[level][Sponge::vbar_sponge].data();
92  }
93 
94  bool l_use_terrain_fitted_coords = (solverChoice.mesh_type != MeshType::ConstantDz);
95  bool l_use_kturb = tc.use_kturb;
96  bool l_use_diff = ( (dc.molec_diff_type != MolecDiffType::None) ||
97  l_use_kturb );
98 
99  const bool use_SurfLayer = (m_SurfaceLayer != nullptr);
100  const MultiFab* z_0 = (use_SurfLayer) ? m_SurfaceLayer->get_z0(level) : nullptr;
101 
102  const BoxArray& ba = state_old[IntVars::cons].boxArray();
103  const BoxArray& ba_z = zvel_old.boxArray();
104  const DistributionMapping& dm = state_old[IntVars::cons].DistributionMap();
105 
106  int num_prim = state_old[IntVars::cons].nComp() - 1;
107 
108  MultiFab S_prim (ba , dm, num_prim, state_old[IntVars::cons].nGrowVect());
109  MultiFab pi_stage (ba , dm, 1, 1);
110  MultiFab fast_coeffs(ba_z, dm, 5, 0);
111 
112  MultiFab* eddyDiffs = eddyDiffs_lev[level].get();
113  MultiFab* SmnSmn = SmnSmn_lev[level].get();
114 
115  // **************************************************************************************
116  // Compute strain for use in slow RHS and Smagorinsky model
117  // **************************************************************************************
118  {
119  BL_PROFILE("erf_advance_strain");
120  if (l_use_diff) {
121 
122  const BCRec* bc_ptr_h = domain_bcs_type.data();
123  const GpuArray<Real, AMREX_SPACEDIM> dxInv = fine_geom.InvCellSizeArray();
124 
125 #ifdef _OPENMP
126 #pragma omp parallel if (Gpu::notInLaunchRegion())
127 #endif
128  for ( MFIter mfi(state_new[IntVars::cons],TileNoZ()); mfi.isValid(); ++mfi)
129  {
130  Box bxcc = mfi.growntilebox(IntVect(1,1,0));
131  Box tbxxy = mfi.tilebox(IntVect(1,1,0),IntVect(1,1,0));
132  Box tbxxz = mfi.tilebox(IntVect(1,0,1),IntVect(1,1,0));
133  Box tbxyz = mfi.tilebox(IntVect(0,1,1),IntVect(1,1,0));
134 
135  if (bxcc.smallEnd(2) != domain.smallEnd(2)) {
136  bxcc.growLo(2,1);
137  tbxxy.growLo(2,1);
138  tbxxz.growLo(2,1);
139  tbxyz.growLo(2,1);
140  }
141 
142  if (bxcc.bigEnd(2) != domain.bigEnd(2)) {
143  bxcc.growHi(2,1);
144  tbxxy.growHi(2,1);
145  tbxxz.growHi(2,1);
146  tbxyz.growHi(2,1);
147  }
148 
149  const Array4<const Real> & u = xvel_old.array(mfi);
150  const Array4<const Real> & v = yvel_old.array(mfi);
151  const Array4<const Real> & w = zvel_old.array(mfi);
152 
153  Array4<Real> tau11 = Tau[level][TauType::tau11].get()->array(mfi);
154  Array4<Real> tau22 = Tau[level][TauType::tau22].get()->array(mfi);
155  Array4<Real> tau33 = Tau[level][TauType::tau33].get()->array(mfi);
156  Array4<Real> tau12 = Tau[level][TauType::tau12].get()->array(mfi);
157  Array4<Real> tau13 = Tau[level][TauType::tau13].get()->array(mfi);
158  Array4<Real> tau23 = Tau[level][TauType::tau23].get()->array(mfi);
159 
160  Array4<Real> tau21 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau21].get()->array(mfi) : Array4<Real>{};
161  Array4<Real> tau31 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau31].get()->array(mfi) : Array4<Real>{};
162  Array4<Real> tau32 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau32].get()->array(mfi) : Array4<Real>{};
163  const Array4<const Real>& z_nd = z_phys_nd[level]->const_array(mfi);
164 
165  const Array4<const Real> mf_mx = mapfac[level][MapFacType::m_x]->const_array(mfi);
166  const Array4<const Real> mf_ux = mapfac[level][MapFacType::u_x]->const_array(mfi);
167  const Array4<const Real> mf_vx = mapfac[level][MapFacType::v_x]->const_array(mfi);
168  const Array4<const Real> mf_my = mapfac[level][MapFacType::m_y]->const_array(mfi);
169  const Array4<const Real> mf_uy = mapfac[level][MapFacType::u_y]->const_array(mfi);
170  const Array4<const Real> mf_vy = mapfac[level][MapFacType::v_y]->const_array(mfi);
171 
172  // We update Tau_corr[level] in erf_make_tau_terms, not here
173  Array4<Real> no_tau_corr_update_here{};
174 
175  if (solverChoice.mesh_type == MeshType::StretchedDz) {
176  ComputeStrain_S(bxcc, tbxxy, tbxxz, tbxyz, domain,
177  u, v, w,
178  tau11, tau22, tau33,
179  tau12, tau21,
180  tau13, tau31,
181  tau23, tau32,
182  stretched_dz_d[level], dxInv,
183  mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
184  no_tau_corr_update_here, no_tau_corr_update_here);
185  } else if (l_use_terrain_fitted_coords) {
186  ComputeStrain_T(bxcc, tbxxy, tbxxz, tbxyz, domain,
187  u, v, w,
188  tau11, tau22, tau33,
189  tau12, tau21,
190  tau13, tau31,
191  tau23, tau32,
192  z_nd, detJ_cc[level]->const_array(mfi), dxInv,
193  mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
194  no_tau_corr_update_here, no_tau_corr_update_here);
195  } else {
196  ComputeStrain_N(bxcc, tbxxy, tbxxz, tbxyz, domain,
197  u, v, w,
198  tau11, tau22, tau33,
199  tau12, tau13, tau23,
200  dxInv,
201  mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
202  no_tau_corr_update_here, no_tau_corr_update_here);
203  }
204  } // mfi
205  } // l_use_diff
206  } // profile
207 
208 #include "ERF_TI_utils.H"
209 
210  // Additional SFS quantities, calculated once per timestep
211  MultiFab* Hfx1 = SFS_hfx1_lev[level].get();
212  MultiFab* Hfx2 = SFS_hfx2_lev[level].get();
213  MultiFab* Hfx3 = SFS_hfx3_lev[level].get();
214  MultiFab* Q1fx1 = SFS_q1fx1_lev[level].get();
215  MultiFab* Q1fx2 = SFS_q1fx2_lev[level].get();
216  MultiFab* Q1fx3 = SFS_q1fx3_lev[level].get();
217  MultiFab* Q2fx3 = SFS_q2fx3_lev[level].get();
218  MultiFab* Diss = SFS_diss_lev[level].get();
219 
220  // *************************************************************************
221  // Calculate cell-centered eddy viscosity & diffusivities
222  //
223  // Notes -- we fill all the data in ghost cells before calling this so
224  // that we can fill the eddy viscosity in the ghost regions and
225  // not have to call a boundary filler on this data itself
226  //
227  // LES - updates both horizontal and vertical eddy viscosity components
228  // PBL - only updates vertical eddy viscosity components so horizontal
229  // components come from the LES model or are left as zero.
230  // *************************************************************************
231  if (l_use_kturb)
232  {
233  // NOTE: state_new transfers to state_old for PBL (due to ptr swap in advance)
234  bool l_use_moisture = ( solverChoice.moisture_type != MoistureType::None );
235  const BCRec* bc_ptr_h = domain_bcs_type.data();
236  ComputeTurbulentViscosity(dt_advance, xvel_old, yvel_old,Tau[level],
237  state_old[IntVars::cons],
238  *walldist[level].get(),
239  *eddyDiffs, *Hfx1, *Hfx2, *Hfx3, *Diss, // to be updated
240  fine_geom, mapfac[level],
241  z_phys_nd[level], solverChoice,
242  m_SurfaceLayer, z_0, l_use_terrain_fitted_coords,
243  l_use_moisture, level,
244  bc_ptr_h);
245  }
246 
247  // ***********************************************************************************************
248  // Update user-defined source terms -- these are defined once per time step (not per RK stage)
249  // ***********************************************************************************************
251  prob->update_rhotheta_sources(old_time,
252  h_rhotheta_src[level], d_rhotheta_src[level],
253  fine_geom, z_phys_cc[level]);
254  }
255 
257  prob->update_rhoqt_sources(old_time,
258  h_rhoqt_src[level], d_rhoqt_src[level],
259  fine_geom, z_phys_cc[level]);
260  }
261 
263  prob->update_geostrophic_profile(old_time,
264  h_u_geos[level], d_u_geos[level],
265  h_v_geos[level], d_v_geos[level],
266  fine_geom, z_phys_cc[level]);
267  }
268 
270  prob->update_w_subsidence(old_time,
271  h_w_subsid[level], d_w_subsid[level],
272  fine_geom, z_phys_nd[level]);
273  }
274 
275  // ***********************************************************************************************
276  // Convert old velocity available on faces to old momentum on faces to be used in time integration
277  // ***********************************************************************************************
278  MultiFab density(state_old[IntVars::cons], make_alias, Rho_comp, 1);
279 
280  //
281  // This is an optimization since we won't need more than one ghost
282  // cell of momentum in the integrator if not using numerical diffusion
283  //
284  IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : xvel_old.nGrowVect();
285  IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : yvel_old.nGrowVect();
286  IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : zvel_old.nGrowVect();
287 
288  VelocityToMomentum(xvel_old, ngu, yvel_old, ngv, zvel_old, ngw, density,
289  state_old[IntVars::xmom],
290  state_old[IntVars::ymom],
291  state_old[IntVars::zmom],
292  domain, domain_bcs_type);
293 
294  MultiFab::Copy(xvel_new,xvel_old,0,0,1,xvel_old.nGrowVect());
295  MultiFab::Copy(yvel_new,yvel_old,0,0,1,yvel_old.nGrowVect());
296  MultiFab::Copy(zvel_new,zvel_old,0,0,1,zvel_old.nGrowVect());
297 
298  bool fast_only = false;
299  bool vel_and_mom_synced = true;
300 
301  apply_bcs(state_old, old_time,
302  state_old[IntVars::cons].nGrow(), state_old[IntVars::xmom].nGrow(),
303  fast_only, vel_and_mom_synced);
304  cons_to_prim(state_old[IntVars::cons], state_old[IntVars::cons].nGrow());
305 
306  // ***********************************************************************************************
307  // Define a new MultiFab that holds q_total and fill it by summing the moisture components --
308  // to be used in buoyancy calculation and as part of the inertial weighting in the
309  // ***********************************************************************************************
310 
311  const bool l_eb_terrain = (solverChoice.terrain_type == TerrainType::EB);
312  MultiFab qt(grids[level], dmap[level], 1, (l_eb_terrain) ? 2 : 1);
313  qt.setVal(0.0);
314 
315 #include "ERF_TI_no_substep_fun.H"
316 #include "ERF_TI_substep_fun.H"
317 #include "ERF_TI_slow_rhs_pre.H"
318 #include "ERF_TI_slow_rhs_post.H"
319 
320  // ***************************************************************************************
321  // Setup the integrator and integrate for a single timestep
322  // **************************************************************************************
323  MRISplitIntegrator<Vector<MultiFab> >& mri_integrator = *mri_integrator_mem[level];
324 
325  // Define rhs and 'post update' utility function that is called after calculating
326  // any state data (e.g. at RK stages or at the end of a timestep)
327  mri_integrator.set_slow_rhs_pre(slow_rhs_fun_pre);
328  mri_integrator.set_slow_rhs_post(slow_rhs_fun_post);
329 
332  mri_integrator.set_no_substep(no_substep_fun);
333 
334  mri_integrator.advance(state_old, state_new, old_time, dt_advance);
335 
336  if (verbose) Print() << "Done with advance_dycore at level " << level << std::endl;
337 }
void ComputeStrain_N(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau13, Array4< Real > &tau23, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_mx, const Array4< const Real > &mf_ux, const Array4< const Real > &mf_vx, const Array4< const Real > &mf_my, const Array4< const Real > &mf_uy, const Array4< const Real > &mf_vy, const BCRec *bc_ptr, Array4< Real > &tau13i, Array4< Real > &tau23i)
Definition: ERF_ComputeStrain_N.cpp:31
void ComputeStrain_S(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau21, Array4< Real > &tau13, Array4< Real > &tau31, Array4< Real > &tau23, Array4< Real > &tau32, const Gpu::DeviceVector< Real > &stretched_dz_d, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_mx, const Array4< const Real > &mf_ux, const Array4< const Real > &mf_vx, const Array4< const Real > &mf_my, const Array4< const Real > &mf_uy, const Array4< const Real > &mf_vy, const BCRec *bc_ptr, Array4< Real > &tau13i, Array4< Real > &tau23i)
Definition: ERF_ComputeStrain_S.cpp:39
void ComputeStrain_T(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau21, Array4< Real > &tau13, Array4< Real > &tau31, Array4< Real > &tau23, Array4< Real > &tau32, const Array4< const Real > &z_nd, const Array4< const Real > &detJ, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_mx, const Array4< const Real > &mf_ux, const Array4< const Real > &mf_vx, const Array4< const Real > &mf_my, const Array4< const Real > &mf_uy, const Array4< const Real > &mf_vy, const BCRec *bc_ptr, Array4< Real > &tau13i, Array4< Real > &tau23i)
Definition: ERF_ComputeStrain_T.cpp:39
void ComputeTurbulentViscosity(Real dt, const MultiFab &xvel, const MultiFab &yvel, Vector< std::unique_ptr< MultiFab >> &Tau_lev, MultiFab &cons_in, const MultiFab &wdist, MultiFab &eddyViscosity, MultiFab &Hfx1, MultiFab &Hfx2, MultiFab &Hfx3, MultiFab &Diss, const Geometry &geom, Vector< std::unique_ptr< MultiFab >> &mapfac, const std::unique_ptr< MultiFab > &z_phys_nd, const SolverChoice &solverChoice, std::unique_ptr< SurfaceLayer > &SurfLayer, const MultiFab *z_0, const bool &use_terrain_fitted_coords, const bool &use_moisture, int level, const BCRec *bc_ptr, bool vert_only)
Definition: ERF_ComputeTurbulentViscosity.cpp:570
@ tau12
Definition: ERF_DataStruct.H:31
@ tau33
Definition: ERF_DataStruct.H:31
@ tau22
Definition: ERF_DataStruct.H:31
@ tau11
Definition: ERF_DataStruct.H:31
@ tau32
Definition: ERF_DataStruct.H:31
@ tau31
Definition: ERF_DataStruct.H:31
@ tau21
Definition: ERF_DataStruct.H:31
@ ubar
Definition: ERF_DataStruct.H:92
@ wbar
Definition: ERF_DataStruct.H:92
@ vbar
Definition: ERF_DataStruct.H:92
@ thetabar
Definition: ERF_DataStruct.H:92
@ nvars_sponge
Definition: ERF_DataStruct.H:97
@ vbar_sponge
Definition: ERF_DataStruct.H:97
@ ubar_sponge
Definition: ERF_DataStruct.H:97
@ v_x
Definition: ERF_DataStruct.H:23
@ u_y
Definition: ERF_DataStruct.H:24
@ v_y
Definition: ERF_DataStruct.H:24
@ m_y
Definition: ERF_DataStruct.H:24
@ u_x
Definition: ERF_DataStruct.H:23
@ m_x
Definition: ERF_DataStruct.H:23
auto no_substep_fun
Definition: ERF_TI_no_substep_fun.H:4
auto slow_rhs_fun_post
Definition: ERF_TI_slow_rhs_post.H:3
auto slow_rhs_fun_pre
Definition: ERF_TI_slow_rhs_pre.H:6
auto acoustic_substepping_fun
Definition: ERF_TI_substep_fun.H:6
auto apply_bcs
Definition: ERF_TI_utils.H:73
auto cons_to_prim
Definition: ERF_TI_utils.H:4
amrex::Vector< std::unique_ptr< amrex::MultiFab > > walldist
Definition: ERF.H:954
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > mapfac
Definition: ERF.H:957
amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > > mri_integrator_mem
Definition: ERF.H:821
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_sinesq_stag_ptrs
Definition: ERF.H:1311
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhotheta_src
Definition: ERF.H:1274
amrex::Vector< amrex::Vector< amrex::Real > > h_w_subsid
Definition: ERF.H:1279
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc
Definition: ERF.H:933
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_sponge_ptrs
Definition: ERF.H:1307
amrex::Vector< amrex::Vector< amrex::Real > > h_rhoqt_src
Definition: ERF.H:1276
amrex::Vector< long > dt_mri_ratio
Definition: ERF.H:806
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q2fx3_lev
Definition: ERF.H:924
static int verbose
Definition: ERF.H:1195
std::unique_ptr< ProblemBase > prob
Definition: ERF.H:793
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > stretched_dz_d
Definition: ERF.H:960
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_diss_lev
Definition: ERF.H:922
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_sinesq_ptrs
Definition: ERF.H:1310
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_v_geos
Definition: ERF.H:1286
amrex::Vector< amrex::Vector< amrex::Real > > h_v_geos
Definition: ERF.H:1285
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhoqt_src
Definition: ERF.H:1277
amrex::Vector< amrex::Vector< amrex::Real > > h_rhotheta_src
Definition: ERF.H:1273
amrex::Vector< amrex::Vector< amrex::Real > > h_u_geos
Definition: ERF.H:1282
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SmnSmn_lev
Definition: ERF.H:908
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_u_geos
Definition: ERF.H:1283
static int fixed_mri_dt_ratio
Definition: ERF.H:1053
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_rayleigh_ptrs
Definition: ERF.H:1306
Definition: ERF_MRI.H:16
void set_acoustic_substepping(std::function< void(int, int, int, T &, const T &, T &, T &, const amrex::Real, const amrex::Real, const amrex::Real, const amrex::Real, const amrex::Real)> F)
Definition: ERF_MRI.H:140
void set_no_substep(std::function< void(T &, T &, T &, amrex::Real, amrex::Real, int)> F)
Definition: ERF_MRI.H:158
void set_slow_rhs_post(std::function< void(T &, T &, T &, T &, const amrex::Real, const amrex::Real, const amrex::Real, const int)> F)
Definition: ERF_MRI.H:135
void set_slow_rhs_pre(std::function< void(T &, T &, T &, const amrex::Real, const amrex::Real, const amrex::Real, const int)> F)
Definition: ERF_MRI.H:131
void set_slow_fast_timestep_ratio(const int timestep_ratio=1)
Definition: ERF_MRI.H:148
amrex::Real advance(T &S_old, T &S_new, amrex::Real time, const amrex::Real time_step)
Definition: ERF_MRI.H:168
@ pi0_comp
Definition: ERF_IndexDefines.H:65
@ p0_comp
Definition: ERF_IndexDefines.H:64
@ r0_comp
Definition: ERF_IndexDefines.H:63
@ qt
Definition: ERF_Kessler.H:27
real(c_double), parameter p0
Definition: ERF_module_model_constants.F90:40
bool rayleigh_damp_V
Definition: ERF_DampingStruct.H:74
bool rayleigh_damp_T
Definition: ERF_DampingStruct.H:76
bool rayleigh_damp_W
Definition: ERF_DampingStruct.H:75
bool rayleigh_damp_U
Definition: ERF_DampingStruct.H:73
Definition: ERF_DiffStruct.H:19
MolecDiffType molec_diff_type
Definition: ERF_DiffStruct.H:84
static MeshType mesh_type
Definition: ERF_DataStruct.H:910
DampingChoice dampingChoice
Definition: ERF_DataStruct.H:920
DiffChoice diffChoice
Definition: ERF_DataStruct.H:919
bool custom_rhotheta_forcing
Definition: ERF_DataStruct.H:997
bool custom_geostrophic_profile
Definition: ERF_DataStruct.H:1000
bool use_num_diff
Definition: ERF_DataStruct.H:1020
bool custom_moisture_forcing
Definition: ERF_DataStruct.H:998
amrex::Vector< TurbChoice > turbChoice
Definition: ERF_DataStruct.H:922
static TerrainType terrain_type
Definition: ERF_DataStruct.H:901
SpongeChoice spongeChoice
Definition: ERF_DataStruct.H:921
Definition: ERF_SpongeStruct.H:15
std::string sponge_type
Definition: ERF_SpongeStruct.H:58
Definition: ERF_TurbStruct.H:41
bool use_kturb
Definition: ERF_TurbStruct.H:396
Here is the call graph for this function:

◆ advance_lsm()

void ERF::advance_lsm ( int  lev,
amrex::MultiFab &  cons_in,
amrex::MultiFab &  xvel_in,
amrex::MultiFab &  yvel_in,
const amrex::Real dt_advance 
)
10 {
11  if (solverChoice.lsm_type != LandSurfaceType::None) {
12  if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
13  lsm.Advance(lev, cons_in, xvel_in, yvel_in, SFS_hfx3_lev[lev].get(), SFS_q1fx3_lev[lev].get(), dt_advance, istep[0]);
14  } else {
15  lsm.Advance(lev, dt_advance);
16  }
17  }
18 }
LandSurface lsm
Definition: ERF.H:872
amrex::Vector< int > istep
Definition: ERF.H:799
void Advance(const int &lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, amrex::MultiFab *hfx3_out, amrex::MultiFab *qfx3_out, const amrex::Real &dt_advance, const int &nstep)
Definition: ERF_LandSurface.H:52
LandSurfaceType lsm_type
Definition: ERF_DataStruct.H:1030

◆ advance_microphysics()

void ERF::advance_microphysics ( int  lev,
amrex::MultiFab &  cons_in,
const amrex::Real dt_advance,
const int &  iteration,
const amrex::Real time 
)
10 {
11  if (solverChoice.moisture_type != MoistureType::None) {
12  micro->Update_Micro_Vars_Lev(lev, cons);
13  micro->Advance(lev, dt_advance, iteration, time, solverChoice, vars_new, z_phys_nd, phys_bc_type);
14  micro->Update_State_Vars_Lev(lev, cons);
15  }
16 }
std::unique_ptr< Microphysics > micro
Definition: ERF.H:856

◆ advance_radiation()

void ERF::advance_radiation ( int  lev,
amrex::MultiFab &  cons_in,
const amrex::Real dt_advance 
)
8 {
9  if (solverChoice.rad_type != RadiationType::None) {
10 #ifdef ERF_USE_NETCDF
11  MultiFab *lat_ptr = lat_m[lev].get();
12  MultiFab *lon_ptr = lon_m[lev].get();
13 #else
14  MultiFab *lat_ptr = nullptr;
15  MultiFab *lon_ptr = nullptr;
16 #endif
17  // RRTMGP inputs names and pointers
18  Vector<std::string> lsm_input_names = rad[lev]->get_lsm_input_varnames();
19  Vector<MultiFab*> lsm_input_ptrs(lsm_input_names.size(),nullptr);
20  for (int i(0); i<lsm_input_ptrs.size(); ++i) {
21  int varIdx = lsm.Get_DataIdx(lev,lsm_input_names[i]);
22  lsm_input_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx);
23  }
24 
25  // RRTMGP output names and pointers
26  Vector<std::string> lsm_output_names = rad[lev]->get_lsm_output_varnames();
27  Vector<MultiFab*> lsm_output_ptrs(lsm_output_names.size(),nullptr);
28  for (int i(0); i<lsm_output_ptrs.size(); ++i) {
29  int varIdx = lsm.Get_DataIdx(lev,lsm_output_names[i]);
30  lsm_output_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx);
31  }
32 
33  // Enter radiation class driver
34  amrex::Real time_for_rad = t_new[lev] + start_time;
35  rad[lev]->Run(lev, istep[lev], time_for_rad, dt_advance,
36  cons.boxArray(), geom[lev], &(cons),
37  sw_lw_fluxes[lev].get(), solar_zenith[lev].get(),
38  lsm_input_ptrs, lsm_output_ptrs,
39  qheating_rates[lev].get(), rad_fluxes[lev].get(),
40  z_phys_nd[lev].get() , lat_ptr, lon_ptr);
41  }
42 }
static amrex::Real start_time
Definition: ERF.H:1033
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sw_lw_fluxes
Definition: ERF.H:890
amrex::Vector< std::unique_ptr< IRadiation > > rad
Definition: ERF.H:878
amrex::Vector< amrex::Real > t_new
Definition: ERF.H:803
amrex::Vector< std::unique_ptr< amrex::MultiFab > > solar_zenith
Definition: ERF.H:891
amrex::Vector< std::unique_ptr< amrex::MultiFab > > lon_m
Definition: ERF.H:756
amrex::Vector< std::unique_ptr< amrex::MultiFab > > lat_m
Definition: ERF.H:756
amrex::Vector< std::unique_ptr< amrex::MultiFab > > qheating_rates
Definition: ERF.H:879
amrex::Vector< std::unique_ptr< amrex::MultiFab > > rad_fluxes
Definition: ERF.H:880
int Get_DataIdx(const int &lev, std::string &varname)
Definition: ERF_LandSurface.H:107
amrex::MultiFab * Get_Data_Ptr(const int &lev, const int &varIdx)
Definition: ERF_LandSurface.H:89
RadiationType rad_type
Definition: ERF_DataStruct.H:1031

◆ appendPlotVariables()

void ERF::appendPlotVariables ( const std::string &  pp_plot_var_names,
amrex::Vector< std::string > &  plot_var_names 
)
private
229 {
230  ParmParse pp(pp_prefix);
231 
232  Vector<std::string> plot_var_names(0);
233  if (pp.contains(pp_plot_var_names.c_str())) {
234  std::string nm;
235  int nPltVars = pp.countval(pp_plot_var_names.c_str());
236  for (int i = 0; i < nPltVars; i++) {
237  pp.get(pp_plot_var_names.c_str(), nm, i);
238  // Add the named variable to our list of plot variables
239  // if it is not already in the list
240  if (!containerHasElement(plot_var_names, nm)) {
241  plot_var_names.push_back(nm);
242  }
243  }
244  }
245 
246  Vector<std::string> tmp_plot_names(0);
247 #ifdef ERF_USE_PARTICLES
248  Vector<std::string> particle_mesh_plot_names;
249  particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
250  for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
251  std::string tmp(particle_mesh_plot_names[i]);
252  if (containerHasElement(plot_var_names, tmp) ) {
253  tmp_plot_names.push_back(tmp);
254  }
255  }
256 #endif
257 
258  {
259  Vector<std::string> microphysics_plot_names;
260  micro->GetPlotVarNames(microphysics_plot_names);
261  if (microphysics_plot_names.size() > 0) {
262  static bool first_call = true;
263  if (first_call) {
264  Print() << getEnumNameString(solverChoice.moisture_type)
265  << ": the following additional variables are available to plot:\n";
266  for (int i = 0; i < microphysics_plot_names.size(); i++) {
267  Print() << " " << microphysics_plot_names[i] << "\n";
268  }
269  first_call = false;
270  }
271  for (auto& plot_name : microphysics_plot_names) {
272  if (containerHasElement(plot_var_names, plot_name)) {
273  tmp_plot_names.push_back(plot_name);
274  }
275  }
276  }
277  }
278 
279  for (int i = 0; i < tmp_plot_names.size(); i++) {
280  a_plot_var_names.push_back( tmp_plot_names[i] );
281  }
282 
283  // Finally, check to see if we found all the requested variables
284  for (const auto& plot_name : plot_var_names) {
285  if (!containerHasElement(a_plot_var_names, plot_name)) {
286  if (amrex::ParallelDescriptor::IOProcessor()) {
287  Warning("\nWARNING: Requested to plot variable '" + plot_name + "' but it is not available");
288  }
289  }
290  }
291 }
bool containerHasElement(const V &iterable, const T &query)
Definition: ERF_Container.H:5
std::string pp_prefix
Definition: ERF.H:526
Here is the call graph for this function:

◆ AverageDown()

void ERF::AverageDown ( )
private
17 {
18  AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
19 
20  int src_comp, num_comp;
21  for (int lev = finest_level-1; lev >= 0; --lev)
22  {
23  // If anelastic we don't average down rho because rho == rho0.
24  if (solverChoice.anelastic[lev]) {
25  src_comp = 1;
26  } else {
27  src_comp = 0;
28  }
29  num_comp = vars_new[0][Vars::cons].nComp() - src_comp;
30  AverageDownTo(lev,src_comp,num_comp);
31  }
32 }
void AverageDownTo(int crse_lev, int scomp, int ncomp)
Definition: ERF_AverageDown.cpp:36
CouplingType coupling_type
Definition: ERF_DataStruct.H:1026

◆ AverageDownTo()

void ERF::AverageDownTo ( int  crse_lev,
int  scomp,
int  ncomp 
)
37 {
38  if (solverChoice.anelastic[crse_lev]) {
39  AMREX_ALWAYS_ASSERT(scomp == 1);
40  } else {
41  AMREX_ALWAYS_ASSERT(scomp == 0);
42  }
43 
44  AMREX_ALWAYS_ASSERT(ncomp == vars_new[crse_lev][Vars::cons].nComp() - scomp);
45  AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
46 
47  // ******************************************************************************************
48  // First do cell-centered quantities
49  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
50  // m is the map scale factor at cell centers
51  // Here we multiply (rho S) by detJ and divide (rho S) by m^2 before average down
52  // ******************************************************************************************
53  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
54  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
55  const Box& bx = mfi.tilebox();
56  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
57  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
58  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
59  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
60  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
61  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
62  {
63  cons_arr(i,j,k,scomp+n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
64  });
65  } else {
66  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
67  {
68  cons_arr(i,j,k,scomp+n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
69  });
70  }
71  } // mfi
72  } // lev
73 
74  int fine_lev = crse_lev+1;
75 
76  if (interpolation_type == StateInterpType::Perturbational) {
77  // Make the fine rho and (rho theta) be perturbational
78  MultiFab::Divide(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
79  Rho_comp,RhoTheta_comp,1,IntVect{0});
80  MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
81  BaseState::r0_comp,Rho_comp,1,IntVect{0});
82  MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
83  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
84 
85  // Make the crse rho and (rho theta) be perturbational
86  MultiFab::Divide(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
87  Rho_comp,RhoTheta_comp,1,IntVect{0});
88  MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
89  BaseState::r0_comp,Rho_comp,1,IntVect{0});
90  MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
91  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
92  }
93 
94  if (SolverChoice::terrain_type != TerrainType::EB) {
95  average_down(vars_new[crse_lev+1][Vars::cons],vars_new[crse_lev ][Vars::cons],
96  scomp, ncomp, refRatio(crse_lev));
97  } else {
98  // const auto dx = geom[fine_lev].CellSize();
99  // Setting cell_vol to the exact value may cause round-off errors in volume average.
100  // const Real cell_vol = dx[0]*dx[1]*dx[2];
101  constexpr Real cell_vol = 1.0;
102  const BoxArray& ba = vars_new[fine_lev][IntVars::cons].boxArray();
103  const DistributionMapping& dm = vars_new[fine_lev][IntVars::cons].DistributionMap();
104  MultiFab vol_fine(ba, dm, 1, 0);
105  vol_fine.setVal(cell_vol);
106  EB_average_down(vars_new[fine_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
107  vol_fine, *detJ_cc[fine_lev],
108  scomp, ncomp, refRatio(crse_lev));
109  }
110 
111  if (interpolation_type == StateInterpType::Perturbational) {
112  // Restore the fine data to what it was
113  MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
114  BaseState::r0_comp,Rho_comp,1,IntVect{0});
115  MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
116  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
117  MultiFab::Multiply(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
118  Rho_comp,RhoTheta_comp,1,IntVect{0});
119 
120  // Make the crse data be full values not perturbational
121  MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
122  BaseState::r0_comp,Rho_comp,1,IntVect{0});
123  MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
124  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
125  MultiFab::Multiply(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
126  Rho_comp,RhoTheta_comp,1,IntVect{0});
127  }
128 
129  vars_new[crse_lev][Vars::cons].FillBoundary(geom[crse_lev].periodicity());
130 
131  // ******************************************************************************************
132  // Here we multiply (rho S) by m^2 and divide by detJ after average down
133  // ******************************************************************************************
134  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
135  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
136  const Box& bx = mfi.tilebox();
137  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
138  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
139  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
140  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
141  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
142  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
143  {
144  cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
145  });
146  } else { // MeshType::ConstantDz
147  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
148  {
149  cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
150  });
151  }
152  } // mfi
153  } // lev
154 
155  // Fill EB covered cells by old values
156  // (This won't be needed because EB_average_down copyies the covered value.)
157  if (SolverChoice::terrain_type == TerrainType::EB) {
158  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
159  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
160  const Box& bx = mfi.tilebox();
161  const Array4< Real> cons_new = vars_new[lev][Vars::cons].array(mfi);
162  const Array4<const Real> cons_old = vars_old[lev][Vars::cons].array(mfi);
163  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
164  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
165  {
166  if (detJ_arr(i,j,k) == 0.0) {
167  cons_new(i,j,k,scomp+n) = cons_old(i,j,k,scomp+n);
168  }
169  });
170  } // mfi
171  } // lev
172  }
173 
174  // ******************************************************************************************
175  // Now average down momenta.
176  // Note that vars_new holds velocities not momenta, but we want to do conservative
177  // averaging so we first convert to momentum, then average down, then convert
178  // back to velocities -- only on the valid region
179  // ******************************************************************************************
180  for (int lev = crse_lev; lev <= crse_lev+1; lev++)
181  {
182  // FillBoundary for density so we can go back and forth between velocity and momentum
183  vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
184 
185  if (SolverChoice::terrain_type != TerrainType::EB) {
186  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect(0,0,0),
187  vars_new[lev][Vars::yvel], IntVect(0,0,0),
188  vars_new[lev][Vars::zvel], IntVect(0,0,0),
189  vars_new[lev][Vars::cons],
190  rU_new[lev],
191  rV_new[lev],
192  rW_new[lev],
193  Geom(lev).Domain(),
195  } else {
196  const MultiFab& c_vfrac = (get_eb(lev).get_const_factory())->getVolFrac();
197 
198  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect(0,0,0),
199  vars_new[lev][Vars::yvel], IntVect(0,0,0),
200  vars_new[lev][Vars::zvel], IntVect(0,0,0),
201  vars_new[lev][Vars::cons],
202  rU_new[lev],
203  rV_new[lev],
204  rW_new[lev],
205  Geom(lev).Domain(),
207  &c_vfrac);
208  }
209  }
210 
211  if (SolverChoice::terrain_type != TerrainType::EB) {
212  average_down_faces(rU_new[crse_lev+1], rU_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
213  average_down_faces(rV_new[crse_lev+1], rV_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
214  average_down_faces(rW_new[crse_lev+1], rW_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
215  } else {
216  EB_average_down_faces({&rU_new[crse_lev+1], &rV_new[crse_lev+1], &rW_new[crse_lev+1]},
217  {&rU_new[crse_lev], &rV_new[crse_lev], &rW_new[crse_lev]},
218  refRatio(crse_lev), 0);
219  }
220 
221  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
222  if (SolverChoice::terrain_type != TerrainType::EB) {
224  vars_new[lev][Vars::yvel],
225  vars_new[lev][Vars::zvel],
226  vars_new[lev][Vars::cons],
227  rU_new[lev],
228  rV_new[lev],
229  rW_new[lev],
230  Geom(lev).Domain(),
232  } else {
233  const MultiFab& c_vfrac = (get_eb(lev).get_const_factory())->getVolFrac();
234 
236  vars_new[lev][Vars::yvel],
237  vars_new[lev][Vars::zvel],
238  vars_new[lev][Vars::cons],
239  rU_new[lev],
240  rV_new[lev],
241  rW_new[lev],
242  Geom(lev).Domain(),
244  &c_vfrac);
245  }
246  }
247 }
void MomentumToVelocity(MultiFab &xvel, MultiFab &yvel, MultiFab &zvel, const MultiFab &density, const MultiFab &xmom_in, const MultiFab &ymom_in, const MultiFab &zmom_in, const Box &domain, const Vector< BCRec > &domain_bcs_type_h, const MultiFab *c_vfrac)
Definition: ERF_MomentumToVelocity.cpp:25
eb_ const & get_eb(int lev) const noexcept
Definition: ERF.H:1622
static StateInterpType interpolation_type
Definition: ERF.H:1212
const std::unique_ptr< amrex::EBFArrayBoxFactory > & get_const_factory() const noexcept
Definition: ERF_EB.H:46
@ th0_comp
Definition: ERF_IndexDefines.H:66
Here is the call graph for this function:

◆ build_fine_mask()

MultiFab & ERF::build_fine_mask ( int  level)

Helper function for constructing a fine mask, that is, a MultiFab masking coarser data at a lower level by zeroing out covered cells in the fine mask MultiFab we compute.

Parameters
levelFine level index which masks underlying coarser data
77 {
78  // Mask for zeroing covered cells
79  AMREX_ASSERT(level > 0);
80 
81  const BoxArray& cba = grids[level-1];
82  const DistributionMapping& cdm = dmap[level-1];
83 
84  // TODO -- we should make a vector of these a member of ERF class
85  fine_mask.define(cba, cdm, 1, 0, MFInfo());
86  fine_mask.setVal(1.0);
87 
88  BoxArray fba = grids[level];
89  iMultiFab ifine_mask = makeFineMask(cba, cdm, fba, ref_ratio[level-1], 1, 0);
90 
91  const auto fma = fine_mask.arrays();
92  const auto ifma = ifine_mask.arrays();
93  ParallelFor(fine_mask, [=] AMREX_GPU_DEVICE(int bno, int i, int j, int k) noexcept
94  {
95  fma[bno](i,j,k) = ifma[bno](i,j,k);
96  });
97 
98  return fine_mask;
99 }
amrex::MultiFab fine_mask
Definition: ERF.H:1343

◆ check_for_low_temp()

void ERF::check_for_low_temp ( amrex::MultiFab &  S)
2838 {
2839  // *****************************************************************************
2840  // Test for low temp (low is defined as beyond the microphysics range of validity)
2841  // *****************************************************************************
2842  //
2843  // This value is defined in erf_dtesati in Source/Utils/ERF_MicrophysicsUtils.H
2844  Real t_low = 273.16 - 85.;
2845  //
2846  for (MFIter mfi(S); mfi.isValid(); ++mfi)
2847  {
2848  Box bx = mfi.tilebox();
2849  const Array4<Real> &s_arr = S.array(mfi);
2850  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
2851  {
2852  const Real rho = s_arr(i, j, k, Rho_comp);
2853  const Real rhotheta = s_arr(i, j, k, RhoTheta_comp);
2854  const Real qv = s_arr(i, j, k, RhoQ1_comp);
2855 
2856  Real temp = getTgivenRandRTh(rho, rhotheta, qv);
2857 
2858  if (temp < t_low) {
2859 #ifdef AMREX_USE_GPU
2860  AMREX_DEVICE_PRINTF("Temperature too low going into microphysics in cell: %d %d %d %e \n", i,j,k,temp);
2861 #else
2862  printf("Temperature too low going into microphyics in cell: %d %d %d \n", i,j,k);
2863  printf("Based on temp / rhotheta / rho %e %e %e \n", temp,rhotheta,rho);
2864  amrex::Abort();
2865 #endif
2866  }
2867  });
2868  }
2869 }
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getTgivenRandRTh(const amrex::Real rho, const amrex::Real rhotheta, const amrex::Real qv=0.0)
Definition: ERF_EOS.H:46
@ rho
Definition: ERF_Kessler.H:22
@ qv
Definition: ERF_Kessler.H:28
Here is the call graph for this function:

◆ check_for_negative_theta()

void ERF::check_for_negative_theta ( amrex::MultiFab &  S)
2873 {
2874  // *****************************************************************************
2875  // Test for negative (rho theta)
2876  // *****************************************************************************
2877  for (MFIter mfi(S); mfi.isValid(); ++mfi)
2878  {
2879  Box bx = mfi.tilebox();
2880  const Array4<Real> &s_arr = S.array(mfi);
2881  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
2882  {
2883  const Real rhotheta = s_arr(i, j, k, RhoTheta_comp);
2884  if (rhotheta <= 0.) {
2885 #ifdef AMREX_USE_GPU
2886  AMREX_DEVICE_PRINTF("RhoTheta is negative at %d %d %d %e \n", i,j,k,rhotheta);
2887 #else
2888  printf("RhoTheta is negative at %d %d %d %e \n", i,j,k,rhotheta);
2889  amrex::Abort("Bad theta in check_for_negative_theta");
2890 #endif
2891  }
2892  });
2893  } // mfi
2894 }

◆ check_state_for_nans()

void ERF::check_state_for_nans ( amrex::MultiFab const &  S)
2788 {
2789  int ncomp = S.nComp();
2790  for (int lev = 0; lev <= finest_level; lev++)
2791  {
2792  //
2793  // Test at the end of every full timestep whether the solution data contains NaNs
2794  //
2795  bool any_have_nans = false;
2796  for (int i = 0; i < ncomp; i++) {
2797  if (S.contains_nan(i,1,0))
2798  {
2799  amrex::Print() << "Component " << i << "of conserved variables contains NaNs" << '\n';
2800  any_have_nans = true;
2801  }
2802  }
2803  if (any_have_nans) {
2804  exit(0);
2805  }
2806  }
2807 }

◆ check_vels_for_nans()

void ERF::check_vels_for_nans ( amrex::MultiFab const &  xvel,
amrex::MultiFab const &  yvel,
amrex::MultiFab const &  zvel 
)
2811 {
2812  //
2813  // Test at the end of every full timestep whether the solution data contains NaNs
2814  //
2815  bool any_have_nans = false;
2816  if (xvel.contains_nan(0,1,0))
2817  {
2818  amrex::Print() << "x-velocity contains NaNs " << '\n';
2819  any_have_nans = true;
2820  }
2821  if (yvel.contains_nan(0,1,0))
2822  {
2823  amrex::Print() << "y-velocity contains NaNs" << '\n';
2824  any_have_nans = true;
2825  }
2826  if (zvel.contains_nan(0,1,0))
2827  {
2828  amrex::Print() << "z-velocity contains NaNs" << '\n';
2829  any_have_nans = true;
2830  }
2831  if (any_have_nans) {
2832  exit(0);
2833  }
2834 }

◆ ClearLevel()

void ERF::ClearLevel ( int  lev)
override
706 {
707  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
708  vars_new[lev][var_idx].clear();
709  vars_old[lev][var_idx].clear();
710  }
711 
712  base_state[lev].clear();
713 
714  rU_new[lev].clear();
715  rU_old[lev].clear();
716  rV_new[lev].clear();
717  rV_old[lev].clear();
718  rW_new[lev].clear();
719  rW_old[lev].clear();
720 
721  if (lev > 0) {
722  zmom_crse_rhs[lev].clear();
723  }
724 
726  pp_inc[lev].clear();
727  }
728  if (solverChoice.anelastic[lev] == 0) {
729  lagged_delta_rt[lev].clear();
730  }
731  avg_xmom[lev].clear();
732  avg_ymom[lev].clear();
733  avg_zmom[lev].clear();
734 
735  // Clears the integrator memory
736  mri_integrator_mem[lev].reset();
737 
738  // Clears the physical boundary condition routines
739  physbcs_cons[lev].reset();
740  physbcs_u[lev].reset();
741  physbcs_v[lev].reset();
742  physbcs_w[lev].reset();
743  physbcs_base[lev].reset();
744 
745  // Clears the flux register array
746  advflux_reg[lev]->reset();
747 }
amrex::Vector< amrex::MultiFab > avg_xmom
Definition: ERF.H:828
amrex::Vector< amrex::MultiFab > pp_inc
Definition: ERF.H:824
amrex::Vector< amrex::MultiFab > lagged_delta_rt
Definition: ERF.H:827
amrex::Vector< amrex::YAFluxRegister * > advflux_reg
Definition: ERF.H:973
amrex::Vector< amrex::MultiFab > avg_ymom
Definition: ERF.H:829
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > > physbcs_base
Definition: ERF.H:837
amrex::Vector< amrex::MultiFab > avg_zmom
Definition: ERF.H:830
@ NumTypes
Definition: ERF_IndexDefines.H:144
bool project_initial_velocity
Definition: ERF_DataStruct.H:989

◆ cloud_fraction()

Real ERF::cloud_fraction ( amrex::Real  time)
452 {
453  BL_PROFILE("ERF::cloud_fraction()");
454 
455  int lev = 0;
456  // This holds all of qc
457  MultiFab qc(vars_new[lev][Vars::cons],make_alias,RhoQ2_comp,1);
458 
459  int direction = 2; // z-direction
460  Box const& domain = geom[lev].Domain();
461 
462  auto const& qc_arr = qc.const_arrays();
463 
464  // qc_2d is an BaseFab<int> holding the max value over the column
465  auto qc_2d = ReduceToPlane<ReduceOpMax,int>(direction, domain, qc,
466  [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) -> int
467  {
468  if (qc_arr[box_no](i,j,k) > 0) {
469  return 1;
470  } else {
471  return 0;
472  }
473  });
474 
475  auto* p = qc_2d.dataPtr();
476 
477  Long numpts = qc_2d.numPts();
478 
479  AMREX_ASSERT(numpts < Long(std::numeric_limits<int>::max));
480 
481 #if 1
482  if (ParallelDescriptor::UseGpuAwareMpi()) {
483  ParallelDescriptor::ReduceIntMax(p,static_cast<int>(numpts));
484  } else {
485  Gpu::PinnedVector<int> hv(numpts);
486  Gpu::copyAsync(Gpu::deviceToHost, p, p+numpts, hv.data());
487  Gpu::streamSynchronize();
488  ParallelDescriptor::ReduceIntMax(hv.data(),static_cast<int>(numpts));
489  Gpu::copyAsync(Gpu::hostToDevice, hv.data(), hv.data()+numpts, p);
490  }
491 
492  // Sum over component 0
493  Long num_cloudy = qc_2d.template sum<RunOn::Device>(0);
494 
495 #else
496  //
497  // We need this if we allow domain decomposition in the vertical
498  // but for now we leave it commented out
499  //
500  Long num_cloudy = Reduce::Sum<Long>(numpts,
501  [=] AMREX_GPU_DEVICE (Long i) -> Long {
502  if (p[i] == 1) {
503  return 1;
504  } else {
505  return 0;
506  }
507  });
508  ParallelDescriptor::ReduceLongSum(num_cloudy);
509 #endif
510 
511  Real num_total = qc_2d.box().d_numPts();
512 
513  Real cloud_frac = num_cloudy / num_total;
514 
515  return cloud_frac;
516 }
#define RhoQ2_comp
Definition: ERF_IndexDefines.H:43
@ qc
Definition: ERF_SatAdj.H:36

◆ compute_divergence()

void ERF::compute_divergence ( int  lev,
amrex::MultiFab &  rhs,
amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM >  rho0_u_const,
amrex::Geometry const &  geom_at_lev 
)

Project the single-level velocity field to enforce incompressibility Note that the level may or may not be level 0.

11 {
12  BL_PROFILE("ERF::compute_divergence()");
13 
14  auto dxInv = geom_at_lev.InvCellSizeArray();
15 
16  // ****************************************************************************
17  // Compute divergence which will form RHS
18  // Note that we replace "rho0w" with the contravariant momentum, Omega
19  // ****************************************************************************
20  if (solverChoice.terrain_type == TerrainType::EB)
21  {
22  bool already_on_centroids = true;
23  EB_computeDivergence(rhs, rho0_u_const, geom_at_lev, already_on_centroids);
24  }
25  else if (SolverChoice::mesh_type == MeshType::ConstantDz)
26  {
27  computeDivergence(rhs, rho0_u_const, geom_at_lev);
28  }
29  else
30  {
31  for ( MFIter mfi(rhs,TilingIfNotGPU()); mfi.isValid(); ++mfi)
32  {
33  Box bx = mfi.tilebox();
34  const Array4<Real const>& rho0u_arr = rho0_u_const[0]->const_array(mfi);
35  const Array4<Real const>& rho0v_arr = rho0_u_const[1]->const_array(mfi);
36  const Array4<Real const>& rho0w_arr = rho0_u_const[2]->const_array(mfi);
37  const Array4<Real >& rhs_arr = rhs.array(mfi);
38 
39  const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
40  const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
41  const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
42  const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
43 
44  if (SolverChoice::mesh_type == MeshType::StretchedDz)
45  {
46  Real* stretched_dz_d_ptr = stretched_dz_d[lev].data();
47  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
48  {
49  Real inv_dz = 1.0/stretched_dz_d_ptr[k];
50  Real mfsq = mf_mx(i,j,0) * mf_my(i,j,0);
51  rhs_arr(i,j,k) = ( (rho0u_arr(i+1,j ,k )/mf_uy(i+1,j,0) - rho0u_arr(i,j,k)/mf_uy(i,j,0)) * dxInv[0]
52  +(rho0v_arr(i ,j+1,k )/mf_vx(i,j+1,0) - rho0v_arr(i,j,k)/mf_vx(i,j,0)) * dxInv[1]
53  +(rho0w_arr(i ,j ,k+1)/mfsq - rho0w_arr(i,j,k)/mfsq ) * inv_dz ) * mfsq;
54  });
55  }
56  else
57  {
58  //
59  // Note we compute the divergence using "rho0w" == Omega
60  //
61  const Array4<Real const>& ax_arr = ax[lev]->const_array(mfi);
62  const Array4<Real const>& ay_arr = ay[lev]->const_array(mfi);
63  const Array4<Real const>& dJ_arr = detJ_cc[lev]->const_array(mfi);
64 
65  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
66  {
67  Real mfsq = mf_mx(i,j,0) * mf_my(i,j,0);
68  rhs_arr(i,j,k) = ( ( ax_arr(i+1,j,k)*rho0u_arr(i+1,j,k)/mf_uy(i+1,j,0)
69  -ax_arr(i ,j,k)*rho0u_arr(i ,j,k)/mf_uy(i ,j,0) ) * dxInv[0]
70  + ( ay_arr(i,j+1,k)*rho0v_arr(i,j+1,k)/mf_vx(i,j+1,0)
71  -ay_arr(i,j ,k)*rho0v_arr(i,j ,k)/mf_vx(i,j ,0) ) * dxInv[1]
72  +( rho0w_arr(i,j,k+1)/mfsq
73  - rho0w_arr(i,j,k )/mfsq ) * dxInv[2] ) * mfsq / dJ_arr(i,j,k);
74  });
75  }
76  } // mfi
77  }
78 }
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax
Definition: ERF.H:934
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay
Definition: ERF.H:935

◆ ComputeDt()

void ERF::ComputeDt ( int  step = -1)
private

Function that calls estTimeStep for each level

12 {
13  Vector<Real> dt_tmp(finest_level+1);
14 
15  for (int lev = 0; lev <= finest_level; ++lev)
16  {
17  dt_tmp[lev] = estTimeStep(lev, dt_mri_ratio[lev]);
18  }
19 
20  ParallelDescriptor::ReduceRealMin(&dt_tmp[0], dt_tmp.size());
21 
22  Real dt_0 = dt_tmp[0];
23  int n_factor = 1;
24  for (int lev = 0; lev <= finest_level; ++lev) {
25  dt_tmp[lev] = amrex::min(dt_tmp[lev], change_max*dt[lev]);
26  n_factor *= nsubsteps[lev];
27  dt_0 = amrex::min(dt_0, n_factor*dt_tmp[lev]);
28  if (step == 0){
29  dt_0 *= init_shrink;
30  if (verbose && init_shrink != 1.0) {
31  Print() << "Timestep 0: shrink initial dt at level " << lev << " by " << init_shrink << std::endl;
32  }
33  }
34  }
35  // Limit dt's by the value of stop_time.
36  const Real eps = 1.e-3*dt_0;
37  if (t_new[0] + dt_0 > stop_time - eps) {
38  dt_0 = stop_time - t_new[0];
39  }
40 
41  dt[0] = dt_0;
42  for (int lev = 1; lev <= finest_level; ++lev) {
43  dt[lev] = dt[lev-1] / nsubsteps[lev];
44  }
45 }
amrex::Real estTimeStep(int lev, long &dt_fast_ratio) const
Definition: ERF_ComputeTimestep.cpp:54
static amrex::Real stop_time
Definition: ERF.H:1034
amrex::Vector< int > nsubsteps
Definition: ERF.H:800
static amrex::Real init_shrink
Definition: ERF.H:1045
static amrex::Real change_max
Definition: ERF.H:1046

◆ ComputeGhostCells()

static AMREX_FORCE_INLINE int ERF::ComputeGhostCells ( const SolverChoice sc)
inlinestaticprivate
1350  {
1351  int ngrow = 0;
1352 
1353  if (sc.use_num_diff)
1354  {
1355  ngrow = 3;
1356  } else {
1357  if (
1364  { ngrow = 3; }
1365  else if (
1372  { ngrow = 3; }
1373  else if (
1382  { ngrow = 3; }
1383  else if (
1392  { ngrow = 4; }
1393  else
1394  {
1395  if (sc.terrain_type == TerrainType::EB){
1396  ngrow = 3;
1397  } else {
1398  ngrow = 2;
1399  }
1400  }
1401  }
1402 
1403  return ngrow;
1404  }
@ Centered_6th
AdvType moistscal_horiz_adv_type
Definition: ERF_AdvStruct.H:423
AdvType dycore_vert_adv_type
Definition: ERF_AdvStruct.H:420
AdvType moistscal_vert_adv_type
Definition: ERF_AdvStruct.H:424
AdvType dryscal_horiz_adv_type
Definition: ERF_AdvStruct.H:421
AdvType dycore_horiz_adv_type
Definition: ERF_AdvStruct.H:419
AdvType dryscal_vert_adv_type
Definition: ERF_AdvStruct.H:422
AdvChoice advChoice
Definition: ERF_DataStruct.H:918

◆ Construct_ERFFillPatchers()

void ERF::Construct_ERFFillPatchers ( int  lev)
private
2690 {
2691  auto& fine_new = vars_new[lev];
2692  auto& crse_new = vars_new[lev-1];
2693  auto& ba_fine = fine_new[Vars::cons].boxArray();
2694  auto& ba_crse = crse_new[Vars::cons].boxArray();
2695  auto& dm_fine = fine_new[Vars::cons].DistributionMap();
2696  auto& dm_crse = crse_new[Vars::cons].DistributionMap();
2697 
2698  int ncomp = vars_new[lev][Vars::cons].nComp();
2699 
2700  FPr_c.emplace_back(ba_fine, dm_fine, geom[lev] ,
2701  ba_crse, dm_crse, geom[lev-1],
2702  -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
2703  FPr_u.emplace_back(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev] ,
2704  convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
2705  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2706  FPr_v.emplace_back(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev] ,
2707  convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
2708  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2709  FPr_w.emplace_back(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev] ,
2710  convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
2711  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2712 }
int cf_set_width
Definition: ERF.H:898

◆ CreateForecastStateMultiFabs()

void ERF::CreateForecastStateMultiFabs ( amrex::Vector< amrex::Vector< amrex::MultiFab >> &  forecast_state)
250 {
251 
252  forecast_state.resize(max_level+1);
253  for (int lev = 0; lev < max_level+1; ++lev) {
254  forecast_state[lev].resize(vars_new[lev].size()+1);
255  for (int comp = 0; comp < vars_new[lev].size(); ++comp) {
256  const MultiFab& src = vars_new[lev][comp];
257  forecast_state[lev][comp].define(src.boxArray(), src.DistributionMap(),
258  src.nComp(), src.nGrow());
259  }
260  int comp = vars_new[lev].size();
261  const MultiFab& src = vars_new[lev][0];
262  forecast_state[lev][comp].define(src.boxArray(), src.DistributionMap(),
263  2, src.nGrow());
264  }
265 }

◆ CreateWeatherDataGeomBoxArrayDistMap()

void ERF::CreateWeatherDataGeomBoxArrayDistMap ( const std::string &  filename,
amrex::Geometry &  geom_weather,
amrex::BoxArray &  nba,
amrex::DistributionMapping &  dm 
)
159 {
160  Vector<Real> latvec_h, lonvec_h, xvec_h, yvec_h, zvec_h;
161  Vector<Real> rho_h, uvel_h, vvel_h, wvel_h, theta_h, qv_h, qc_h, qr_h;
162 
163  ReadCustomBinaryIC(filename, latvec_h, lonvec_h,
164  xvec_h, yvec_h, zvec_h, rho_h,
165  uvel_h, vvel_h, wvel_h,
166  theta_h, qv_h, qc_h, qr_h);
167 
168  const auto prob_lo_erf = geom[0].ProbLoArray();
169  const auto prob_hi_erf = geom[0].ProbHiArray();
170  const auto dx_erf = geom[0].CellSizeArray();
171 
172  if(prob_lo_erf[0] < xvec_h.front() + 4*dx_erf[0]){
173  amrex::Abort("The xlo value of the domain has to be greater than " + std::to_string(xvec_h.front() + 4*dx_erf[0]));
174  }
175  if(prob_hi_erf[0] > xvec_h.back() - 4*dx_erf[0]){
176  amrex::Abort("The xhi value of the domain has to be less than " + std::to_string(xvec_h.back() - 4*dx_erf[0]));
177  }
178  if(prob_lo_erf[1] < yvec_h.front() + 4*dx_erf[1]){
179  amrex::Abort("The ylo value of the domain has to be greater than " + std::to_string(yvec_h.front() + 4*dx_erf[1]));
180  }
181  if(prob_hi_erf[1] > yvec_h.back() - 4*dx_erf[1]){
182  amrex::Abort("The yhi value of the domain has to be less than " + std::to_string(yvec_h.back() - 4*dx_erf[1]));
183  }
184 
185  // Number of cells
186  int nx_cells = xvec_h.size()-1;
187  int ny_cells = yvec_h.size()-1;
188 
189  const amrex::Geometry& geom0 = geom[0]; // or whatever your Geometry vector is called
190  const amrex::Box& domainBox = geom0.Domain();
191  const amrex::IntVect& domainSize = domainBox.size(); // Number of cells in each direction
192  int nz_cells = domainSize[2];
193 
194  IntVect dom_lo(0, 0, 0);
195  IntVect dom_hi(nx_cells-1, ny_cells-1, nz_cells-1);
196  Box domain(dom_lo, dom_hi);
197 
198  const amrex::Real* prob_hi = geom0.ProbHi();
199 
200  // Define the extents of the physical domain box
201  RealBox real_box({xvec_h[0], yvec_h[0], zvec_h[0]}, {xvec_h[nx_cells], yvec_h[ny_cells], prob_hi[2]});
202 
203  int coord = 0; // Cartesian
204  Array<int, AMREX_SPACEDIM> is_periodic{0, 0, 0}; // non-periodic
205 
206  geom_weather.define(domain, real_box, coord, is_periodic);
207 
208  BoxArray ba(domain);
209  ba.maxSize(64);
210  nba = amrex::convert(ba, IntVect::TheNodeVector()); // nodal in all directions
211 
212  // Create DistributionMapping
213  dm = DistributionMapping(nba);
214  }
void ReadCustomBinaryIC(const std::string filename, amrex::Vector< amrex::Real > &latvec_h, amrex::Vector< amrex::Real > &lonvec_h, amrex::Vector< amrex::Real > &xvec_h, amrex::Vector< amrex::Real > &yvec_h, amrex::Vector< amrex::Real > &zvec_h, amrex::Vector< amrex::Real > &rho_h, amrex::Vector< amrex::Real > &uvel_h, amrex::Vector< amrex::Real > &vvel_h, amrex::Vector< amrex::Real > &wvel_h, amrex::Vector< amrex::Real > &theta_h, amrex::Vector< amrex::Real > &qv_h, amrex::Vector< amrex::Real > &qc_h, amrex::Vector< amrex::Real > &qr_h)
Definition: ERF_ReadCustomBinaryIC.H:12
const auto & dom_hi
Definition: ERF_SetupVertDiff.H:2
const auto & dom_lo
Definition: ERF_SetupVertDiff.H:1
Here is the call graph for this function:

◆ DataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DataLog ( int  i)
inlineprivate
1415  {
1416  return *datalog[i];
1417  }
amrex::Vector< std::unique_ptr< std::fstream > > datalog
Definition: ERF.H:1594

◆ DataLogName()

std::string ERF::DataLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith datalog file.

1610 { return datalogname[i]; }
amrex::Vector< std::string > datalogname
Definition: ERF.H:1597

◆ Define_ERFFillPatchers()

void ERF::Define_ERFFillPatchers ( int  lev)
private
2716 {
2717  auto& fine_new = vars_new[lev];
2718  auto& crse_new = vars_new[lev-1];
2719  auto& ba_fine = fine_new[Vars::cons].boxArray();
2720  auto& ba_crse = crse_new[Vars::cons].boxArray();
2721  auto& dm_fine = fine_new[Vars::cons].DistributionMap();
2722  auto& dm_crse = crse_new[Vars::cons].DistributionMap();
2723 
2724  int ncomp = fine_new[Vars::cons].nComp();
2725 
2726  FPr_c[lev-1].Define(ba_fine, dm_fine, geom[lev] ,
2727  ba_crse, dm_crse, geom[lev-1],
2728  -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
2729  FPr_u[lev-1].Define(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev] ,
2730  convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
2731  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2732  FPr_v[lev-1].Define(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev] ,
2733  convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
2734  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2735  FPr_w[lev-1].Define(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev] ,
2736  convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
2737  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2738 }

◆ DerDataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DerDataLog ( int  i)
inlineprivate
1422  {
1423  return *der_datalog[i];
1424  }
amrex::Vector< std::unique_ptr< std::fstream > > der_datalog
Definition: ERF.H:1595

◆ DerDataLogName()

std::string ERF::DerDataLogName ( int  i) const
inlineprivatenoexcept
1611 { return der_datalogname[i]; }
amrex::Vector< std::string > der_datalogname
Definition: ERF.H:1598

◆ derive_diag_profiles()

void ERF::derive_diag_profiles ( amrex::Real  time,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_u,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_v,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_w,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_rho,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_th,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ksgs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Kmv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Khv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qi,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qg,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ww,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_thth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ku,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_p,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wthv 
)

Computes the profiles for diagnostic quantities.

Parameters
h_avg_uProfile for x-velocity on Host
h_avg_vProfile for y-velocity on Host
h_avg_wProfile for z-velocity on Host
h_avg_rhoProfile for density on Host
h_avg_thProfile for potential temperature on Host
h_avg_ksgsProfile for Kinetic Energy on Host
h_avg_uuProfile for x-velocity squared on Host
h_avg_uvProfile for x-velocity * y-velocity on Host
h_avg_uwProfile for x-velocity * z-velocity on Host
h_avg_vvProfile for y-velocity squared on Host
h_avg_vwProfile for y-velocity * z-velocity on Host
h_avg_wwProfile for z-velocity squared on Host
h_avg_uthProfile for x-velocity * potential temperature on Host
h_avg_uiuiuProfile for u_i*u_i*u triple product on Host
h_avg_uiuivProfile for u_i*u_i*v triple product on Host
h_avg_uiuiwProfile for u_i*u_i*w triple product on Host
h_avg_pProfile for pressure perturbation on Host
h_avg_puProfile for pressure perturbation * x-velocity on Host
h_avg_pvProfile for pressure perturbation * y-velocity on Host
h_avg_pwProfile for pressure perturbation * z-velocity on Host
205 {
206  // We assume that this is always called at level 0
207  int lev = 0;
208 
209  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
210  bool l_use_KE = solverChoice.turbChoice[lev].use_tke;
211  // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, uw, vv, vw, ww, uth, vth, wth,
212  // 0 1 2 3 4 5 6 7 8 9 10 11 12 13
213  // thth, uiuiu, uiuiv, uiuiw, p, pu, pv, pw, qv, qc, qr, wqv, wqc, wqr,
214  // 14 15 16 17 18 19 20 21 22 23 24 25 26 27
215  // qi, qs, qg, wthv
216  // 28 29 30 31
217  MultiFab mf_out(grids[lev], dmap[lev], 32, 0);
218 
219  MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
220 
221  MultiFab u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
222  MultiFab v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
223  MultiFab w_cc(mf_vels, make_alias, 2, 1); // w at cell centers
224 
225  average_face_to_cellcenter(mf_vels,0,
226  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
227 
228  int zdir = 2;
229  auto domain = geom[0].Domain();
230 
231  // Sum in the horizontal plane
232  h_avg_u = sumToLine(mf_vels ,0,1,domain,zdir);
233  h_avg_v = sumToLine(mf_vels ,1,1,domain,zdir);
234  h_avg_w = sumToLine(mf_vels ,2,1,domain,zdir);
235 
236  int hu_size = h_avg_u.size();
237 
238  // Divide by the total number of cells we are averaging over
239  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
240  for (int k = 0; k < hu_size; ++k) {
241  h_avg_u[k] /= area_z; h_avg_v[k] /= area_z; h_avg_w[k] /= area_z;
242  }
243 
244  Gpu::DeviceVector<Real> d_avg_u(hu_size, Real(0.0));
245  Gpu::DeviceVector<Real> d_avg_v(hu_size, Real(0.0));
246  Gpu::DeviceVector<Real> d_avg_w(hu_size, Real(0.0));
247 
248 #if 0
249  auto* avg_u_ptr = d_avg_u.data();
250  auto* avg_v_ptr = d_avg_v.data();
251  auto* avg_w_ptr = d_avg_w.data();
252 #endif
253 
254  Gpu::copy(Gpu::hostToDevice, h_avg_u.begin(), h_avg_u.end(), d_avg_u.begin());
255  Gpu::copy(Gpu::hostToDevice, h_avg_v.begin(), h_avg_v.end(), d_avg_v.begin());
256  Gpu::copy(Gpu::hostToDevice, h_avg_w.begin(), h_avg_w.end(), d_avg_w.begin());
257 
258  int nvars = vars_new[lev][Vars::cons].nComp();
259  MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
260 
261  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
262 
263  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
264 
265  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
266  {
267  const Box& bx = mfi.tilebox();
268  const Array4<Real>& fab_arr = mf_out.array(mfi);
269  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
270  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
271  const Array4<Real>& w_cc_arr = w_cc.array(mfi);
272  const Array4<Real>& cons_arr = mf_cons.array(mfi);
273  const Array4<Real>& p0_arr = p_hse.array(mfi);
274  const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
275  Array4<const Real>{};
276 
277  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
278  {
279  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
280  fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
281  fab_arr(i, j, k, 1) = theta;
282  Real ksgs = 0.0;
283  if (l_use_KE) {
284  ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
285  }
286  fab_arr(i, j, k, 2) = ksgs;
287 #if 1
288  if (l_use_kturb) {
289  fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
290  fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
291  } else {
292  fab_arr(i, j, k, 3) = 0.0;
293  fab_arr(i, j, k, 4) = 0.0;
294  }
295 #else
296  // Here we hijack the "Kturb" variable name to print out the resolved kinetic energy
297  Real upert = u_cc_arr(i,j,k) - avg_u_ptr[k];
298  Real vpert = v_cc_arr(i,j,k) - avg_v_ptr[k];
299  Real wpert = w_cc_arr(i,j,k) - avg_w_ptr[k];
300  fab_arr(i, j, k, 3) = 0.5 * (upert*upert + vpert*vpert + wpert*wpert);
301 #endif
302  fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k); // u*u
303  fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k); // u*v
304  fab_arr(i, j, k, 7) = u_cc_arr(i,j,k) * w_cc_arr(i,j,k); // u*w
305  fab_arr(i, j, k, 8) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k); // v*v
306  fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * w_cc_arr(i,j,k); // v*w
307  fab_arr(i, j, k,10) = w_cc_arr(i,j,k) * w_cc_arr(i,j,k); // w*w
308  fab_arr(i, j, k,11) = u_cc_arr(i,j,k) * theta; // u*th
309  fab_arr(i, j, k,12) = v_cc_arr(i,j,k) * theta; // v*th
310  fab_arr(i, j, k,13) = w_cc_arr(i,j,k) * theta; // w*th
311  fab_arr(i, j, k,14) = theta * theta; // th*th
312 
313  // if the number of fields is changed above, then be sure to update
314  // the following def!
315  Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,8) + fab_arr(i,j,k,10);
316  fab_arr(i, j, k,15) = uiui * u_cc_arr(i,j,k); // (ui*ui)*u
317  fab_arr(i, j, k,16) = uiui * v_cc_arr(i,j,k); // (ui*ui)*v
318  fab_arr(i, j, k,17) = uiui * w_cc_arr(i,j,k); // (ui*ui)*w
319 
320  if (!use_moisture) {
321  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
322  p -= p0_arr(i,j,k);
323  fab_arr(i, j, k,18) = p; // p
324  fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k); // p*u
325  fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k); // p*v
326  fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k); // p*w
327  fab_arr(i, j, k,22) = 0.; // qv
328  fab_arr(i, j, k,23) = 0.; // qc
329  fab_arr(i, j, k,24) = 0.; // qr
330  fab_arr(i, j, k,25) = 0.; // w*qv
331  fab_arr(i, j, k,26) = 0.; // w*qc
332  fab_arr(i, j, k,27) = 0.; // w*qr
333  fab_arr(i, j, k,28) = 0.; // qi
334  fab_arr(i, j, k,29) = 0.; // qs
335  fab_arr(i, j, k,30) = 0.; // qg
336  fab_arr(i, j, k,31) = 0.; // w*thv
337  }
338  });
339  } // mfi
340 
341  if (use_moisture)
342  {
343  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
344 
345  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
346  {
347  const Box& bx = mfi.tilebox();
348  const Array4<Real>& fab_arr = mf_out.array(mfi);
349  const Array4<Real>& cons_arr = mf_cons.array(mfi);
350  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
351  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
352  const Array4<Real>& w_cc_arr = w_cc.array(mfi);
353  const Array4<Real>& p0_arr = p_hse.array(mfi);
354 
355  int rhoqr_comp = solverChoice.moisture_indices.qr;
356 
357  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
358  {
359  Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
360  Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
361  Real qr = (rhoqr_comp > -1) ? cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
362  Real(0.0);
363  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
364 
365  p -= p0_arr(i,j,k);
366  fab_arr(i, j, k,18) = p; // p
367  fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k); // p*u
368  fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k); // p*v
369  fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k); // p*w
370  fab_arr(i, j, k,22) = qv; // qv
371  fab_arr(i, j, k,23) = qc; // qc
372  fab_arr(i, j, k,24) = qr; // qr
373  fab_arr(i, j, k,25) = w_cc_arr(i,j,k) * qv; // w*qv
374  fab_arr(i, j, k,26) = w_cc_arr(i,j,k) * qc; // w*qc
375  fab_arr(i, j, k,27) = w_cc_arr(i,j,k) * qr; // w*qr
376  if (n_qstate_moist > 3) {
377  fab_arr(i, j, k,28) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp); // qi
378  fab_arr(i, j, k,29) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp); // qs
379  fab_arr(i, j, k,30) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp); // qg
380  } else {
381  fab_arr(i, j, k,28) = 0.0; // qi
382  fab_arr(i, j, k,29) = 0.0; // qs
383  fab_arr(i, j, k,30) = 0.0; // qg
384  }
385  Real ql = qc + qr;
386  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
387  Real thv = theta * (1 + 0.61*qv - ql);
388  fab_arr(i, j, k,31) = w_cc_arr(i,j,k) * thv; // w*thv
389  });
390  } // mfi
391  } // use_moisture
392 
393  h_avg_rho = sumToLine(mf_out, 0,1,domain,zdir);
394  h_avg_th = sumToLine(mf_out, 1,1,domain,zdir);
395  h_avg_ksgs = sumToLine(mf_out, 2,1,domain,zdir);
396  h_avg_Kmv = sumToLine(mf_out, 3,1,domain,zdir);
397  h_avg_Khv = sumToLine(mf_out, 4,1,domain,zdir);
398  h_avg_uu = sumToLine(mf_out, 5,1,domain,zdir);
399  h_avg_uv = sumToLine(mf_out, 6,1,domain,zdir);
400  h_avg_uw = sumToLine(mf_out, 7,1,domain,zdir);
401  h_avg_vv = sumToLine(mf_out, 8,1,domain,zdir);
402  h_avg_vw = sumToLine(mf_out, 9,1,domain,zdir);
403  h_avg_ww = sumToLine(mf_out,10,1,domain,zdir);
404  h_avg_uth = sumToLine(mf_out,11,1,domain,zdir);
405  h_avg_vth = sumToLine(mf_out,12,1,domain,zdir);
406  h_avg_wth = sumToLine(mf_out,13,1,domain,zdir);
407  h_avg_thth = sumToLine(mf_out,14,1,domain,zdir);
408  h_avg_uiuiu = sumToLine(mf_out,15,1,domain,zdir);
409  h_avg_uiuiv = sumToLine(mf_out,16,1,domain,zdir);
410  h_avg_uiuiw = sumToLine(mf_out,17,1,domain,zdir);
411  h_avg_p = sumToLine(mf_out,18,1,domain,zdir);
412  h_avg_pu = sumToLine(mf_out,19,1,domain,zdir);
413  h_avg_pv = sumToLine(mf_out,20,1,domain,zdir);
414  h_avg_pw = sumToLine(mf_out,21,1,domain,zdir);
415  h_avg_qv = sumToLine(mf_out,22,1,domain,zdir);
416  h_avg_qc = sumToLine(mf_out,23,1,domain,zdir);
417  h_avg_qr = sumToLine(mf_out,24,1,domain,zdir);
418  h_avg_wqv = sumToLine(mf_out,25,1,domain,zdir);
419  h_avg_wqc = sumToLine(mf_out,26,1,domain,zdir);
420  h_avg_wqr = sumToLine(mf_out,27,1,domain,zdir);
421  h_avg_qi = sumToLine(mf_out,28,1,domain,zdir);
422  h_avg_qs = sumToLine(mf_out,29,1,domain,zdir);
423  h_avg_qg = sumToLine(mf_out,30,1,domain,zdir);
424  h_avg_wthv = sumToLine(mf_out,31,1,domain,zdir);
425 
426  // Divide by the total number of cells we are averaging over
427  int h_avg_u_size = static_cast<int>(h_avg_u.size());
428  for (int k = 0; k < h_avg_u_size; ++k) {
429  h_avg_rho[k] /= area_z;
430  h_avg_ksgs[k] /= area_z;
431  h_avg_Kmv[k] /= area_z;
432  h_avg_Khv[k] /= area_z;
433  h_avg_th[k] /= area_z;
434  h_avg_thth[k] /= area_z;
435  h_avg_uu[k] /= area_z;
436  h_avg_uv[k] /= area_z;
437  h_avg_uw[k] /= area_z;
438  h_avg_vv[k] /= area_z;
439  h_avg_vw[k] /= area_z;
440  h_avg_ww[k] /= area_z;
441  h_avg_uth[k] /= area_z;
442  h_avg_vth[k] /= area_z;
443  h_avg_wth[k] /= area_z;
444  h_avg_uiuiu[k] /= area_z;
445  h_avg_uiuiv[k] /= area_z;
446  h_avg_uiuiw[k] /= area_z;
447  h_avg_p[k] /= area_z;
448  h_avg_pu[k] /= area_z;
449  h_avg_pv[k] /= area_z;
450  h_avg_pw[k] /= area_z;
451  h_avg_qv[k] /= area_z;
452  h_avg_qc[k] /= area_z;
453  h_avg_qr[k] /= area_z;
454  h_avg_wqv[k] /= area_z;
455  h_avg_wqc[k] /= area_z;
456  h_avg_wqr[k] /= area_z;
457  h_avg_qi[k] /= area_z;
458  h_avg_qs[k] /= area_z;
459  h_avg_qg[k] /= area_z;
460  h_avg_wthv[k] /= area_z;
461  }
462 
463 #if 0
464  // Here we print the integrated total kinetic energy as computed in the 1D profile above
465  Real sum = 0.;
466  Real dz = geom[0].ProbHi(2) / static_cast<Real>(h_avg_u_size);
467  for (int k = 0; k < h_avg_u_size; ++k) {
468  sum += h_avg_kturb[k] * h_avg_rho[k] * dz;
469  }
470  amrex::Print() << "ITKE " << time << " " << sum << " using " << h_avg_u_size << " " << dz << std::endl;
471 #endif
472 }
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getPgivenRTh(const amrex::Real rhotheta, const amrex::Real qv=0.)
Definition: ERF_EOS.H:81
#define RhoQ3_comp
Definition: ERF_IndexDefines.H:44
#define RhoQ6_comp
Definition: ERF_IndexDefines.H:47
#define RhoQ5_comp
Definition: ERF_IndexDefines.H:46
#define RhoKE_comp
Definition: ERF_IndexDefines.H:38
@ Theta_v
Definition: ERF_IndexDefines.H:176
@ Mom_v
Definition: ERF_IndexDefines.H:175
@ theta
Definition: ERF_MM5.H:20
Here is the call graph for this function:

◆ derive_diag_profiles_stag()

void ERF::derive_diag_profiles_stag ( amrex::Real  time,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_u,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_v,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_w,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_rho,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_th,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ksgs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Kmv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Khv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qi,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qg,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ww,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_thth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ku,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_p,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wthv 
)

Computes the profiles for diagnostic quantities at staggered heights.

Parameters
h_avg_uProfile for x-velocity on Host
h_avg_vProfile for y-velocity on Host
h_avg_wProfile for z-velocity on Host
h_avg_rhoProfile for density on Host
h_avg_thProfile for potential temperature on Host
h_avg_ksgsProfile for Kinetic Energy on Host
h_avg_uuProfile for x-velocity squared on Host
h_avg_uvProfile for x-velocity * y-velocity on Host
h_avg_uwProfile for x-velocity * z-velocity on Host
h_avg_vvProfile for y-velocity squared on Host
h_avg_vwProfile for y-velocity * z-velocity on Host
h_avg_wwProfile for z-velocity squared on Host
h_avg_uthProfile for x-velocity * potential temperature on Host
h_avg_uiuiuProfile for u_i*u_i*u triple product on Host
h_avg_uiuivProfile for u_i*u_i*v triple product on Host
h_avg_uiuiwProfile for u_i*u_i*w triple product on Host
h_avg_pProfile for pressure perturbation on Host
h_avg_puProfile for pressure perturbation * x-velocity on Host
h_avg_pvProfile for pressure perturbation * y-velocity on Host
h_avg_pwProfile for pressure perturbation * z-velocity on Host
311 {
312  // We assume that this is always called at level 0
313  int lev = 0;
314 
315  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
316  bool l_use_KE = solverChoice.turbChoice[lev].use_tke;
317  // Note: "uiui" == u_i*u_i = u*u + v*v + w*w
318  // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, vv, uth, vth,
319  // indices: 0 1 2 3 4 5 6 7 8 9
320  // thth, uiuiu, uiuiv, p, pu, pv, qv, qc, qr, qi, qs, qg
321  // 10 11 12 13 14 15 16 17 18 19 20 21
322  MultiFab mf_out(grids[lev], dmap[lev], 22, 0);
323 
324  // This will hold uw, vw, ww, wth, uiuiw, pw, wqv, wqc, wqr, wthv
325  // indices: 0 1 2 3 4 5 6 7 8 9
326  MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 10, 0);
327 
328  // This is only used to average u and v; w is not averaged to cell centers
329  MultiFab mf_vels(grids[lev], dmap[lev], 2, 0);
330 
331  MultiFab u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
332  MultiFab v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
333  MultiFab w_fc(vars_new[lev][Vars::zvel], make_alias, 0, 1); // w at face centers (staggered)
334 
335  int zdir = 2;
336  auto domain = geom[0].Domain();
337  Box stag_domain = domain;
338  stag_domain.convert(IntVect(0,0,1));
339 
340  int nvars = vars_new[lev][Vars::cons].nComp();
341  MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
342 
343  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
344 
345  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
346 
347  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
348  {
349  const Box& bx = mfi.tilebox();
350  const Array4<Real>& fab_arr = mf_out.array(mfi);
351  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
352  const Array4<Real>& u_arr = vars_new[lev][Vars::xvel].array(mfi);
353  const Array4<Real>& v_arr = vars_new[lev][Vars::yvel].array(mfi);
354  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
355  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
356  const Array4<Real>& w_fc_arr = w_fc.array(mfi);
357  const Array4<Real>& cons_arr = mf_cons.array(mfi);
358  const Array4<Real>& p0_arr = p_hse.array(mfi);
359  const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
360  Array4<const Real>{};
361 
362  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
363  {
364  u_cc_arr(i,j,k) = 0.5 * (u_arr(i,j,k) + u_arr(i+1,j ,k));
365  v_cc_arr(i,j,k) = 0.5 * (v_arr(i,j,k) + v_arr(i ,j+1,k));
366 
367  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
368  fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
369  fab_arr(i, j, k, 1) = theta;
370  Real ksgs = 0.0;
371  if (l_use_KE) {
372  ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
373  }
374  fab_arr(i, j, k, 2) = ksgs;
375  if (l_use_kturb) {
376  fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
377  fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
378  } else {
379  fab_arr(i, j, k, 3) = 0.0;
380  fab_arr(i, j, k, 4) = 0.0;
381  }
382  fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k); // u*u
383  fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k); // u*v
384  fab_arr(i, j, k, 7) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k); // v*v
385  fab_arr(i, j, k, 8) = u_cc_arr(i,j,k) * theta; // u*th
386  fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * theta; // v*th
387  fab_arr(i, j, k,10) = theta * theta; // th*th
388 
389  Real wcc = 0.5 * (w_fc_arr(i,j,k) + w_fc_arr(i,j,k+1));
390 
391  // if the number of fields is changed above, then be sure to update
392  // the following def!
393  Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,7) + wcc*wcc;
394  fab_arr(i, j, k,11) = uiui * u_cc_arr(i,j,k); // (ui*ui)*u
395  fab_arr(i, j, k,12) = uiui * v_cc_arr(i,j,k); // (ui*ui)*v
396 
397  if (!use_moisture) {
398  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
399  p -= p0_arr(i,j,k);
400  fab_arr(i, j, k,13) = p; // p
401  fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k); // p*u
402  fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k); // p*v
403  fab_arr(i, j, k,16) = 0.; // qv
404  fab_arr(i, j, k,17) = 0.; // qc
405  fab_arr(i, j, k,18) = 0.; // qr
406  fab_arr(i, j, k,19) = 0.; // qi
407  fab_arr(i, j, k,20) = 0.; // qs
408  fab_arr(i, j, k,21) = 0.; // qg
409  }
410  });
411 
412  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
413  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
414  {
415  // average to z faces (first to cell centers, then in z)
416  Real uface = 0.25 * ( u_arr(i ,j,k) + u_arr(i ,j,k-1)
417  + u_arr(i+1,j,k) + u_arr(i+1,j,k-1));
418  Real vface = 0.25 * ( v_arr(i,j ,k) + v_arr(i,j ,k-1)
419  + v_arr(i,j+1,k) + v_arr(i,j+1,k-1));
420  Real theta0 = cons_arr(i,j,k ,RhoTheta_comp) / cons_arr(i,j,k ,Rho_comp);
421  Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
422  Real thface = 0.5*(theta0 + theta1);
423  fab_arr_stag(i,j,k,0) = uface * w_fc_arr(i,j,k); // u*w
424  fab_arr_stag(i,j,k,1) = vface * w_fc_arr(i,j,k); // v*w
425  fab_arr_stag(i,j,k,2) = w_fc_arr(i,j,k) * w_fc_arr(i,j,k); // w*w
426  fab_arr_stag(i,j,k,3) = thface * w_fc_arr(i,j,k); // th*w
427  Real uiui = uface*uface + vface*vface + fab_arr_stag(i,j,k,2);
428  fab_arr_stag(i,j,k,4) = uiui * w_fc_arr(i,j,k); // (ui*ui)*w
429  if (!use_moisture) {
430  Real p0 = getPgivenRTh(cons_arr(i, j, k , RhoTheta_comp)) - p0_arr(i,j,k );
431  Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp)) - p0_arr(i,j,k-1);
432  Real pface = 0.5 * (p0 + p1);
433  fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k); // p*w
434  fab_arr_stag(i,j,k,6) = 0.; // w*qv
435  fab_arr_stag(i,j,k,7) = 0.; // w*qc
436  fab_arr_stag(i,j,k,8) = 0.; // w*qr
437  fab_arr_stag(i,j,k,9) = 0.; // w*thv
438  }
439  });
440 
441  } // mfi
442 
443  if (use_moisture)
444  {
445  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
446 
447  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
448  {
449  const Box& bx = mfi.tilebox();
450  const Array4<Real>& fab_arr = mf_out.array(mfi);
451  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
452  const Array4<Real>& cons_arr = mf_cons.array(mfi);
453  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
454  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
455  const Array4<Real>& w_fc_arr = w_fc.array(mfi);
456  const Array4<Real>& p0_arr = p_hse.array(mfi);
457 
458  int rhoqr_comp = solverChoice.moisture_indices.qr;
459 
460  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
461  {
462  Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
463  Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
464  Real qr = (rhoqr_comp > -1) ? cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
465  Real(0.0);
466  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
467 
468  p -= p0_arr(i,j,k);
469  fab_arr(i, j, k,13) = p; // p
470  fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k); // p*u
471  fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k); // p*v
472  fab_arr(i, j, k,16) = qv; // qv
473  fab_arr(i, j, k,17) = qc; // qc
474  fab_arr(i, j, k,18) = qr; // qr
475  if (n_qstate_moist > 3) { // SAM model
476  fab_arr(i, j, k,19) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp); // qi
477  fab_arr(i, j, k,20) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp); // qs
478  fab_arr(i, j, k,21) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp); // qg
479  } else {
480  fab_arr(i, j, k,19) = 0.0; // qi
481  fab_arr(i, j, k,20) = 0.0; // qs
482  fab_arr(i, j, k,21) = 0.0; // qg
483  }
484  });
485 
486  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
487  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
488  {
489  Real qv0 = cons_arr(i,j,k ,RhoQ1_comp) / cons_arr(i,j,k ,Rho_comp);
490  Real qv1 = cons_arr(i,j,k-1,RhoQ1_comp) / cons_arr(i,j,k-1,Rho_comp);
491  Real qc0 = cons_arr(i,j,k ,RhoQ2_comp) / cons_arr(i,j,k ,Rho_comp);
492  Real qc1 = cons_arr(i,j,k-1,RhoQ2_comp) / cons_arr(i,j,k-1,Rho_comp);
493  Real qr0 = (rhoqr_comp > -1) ? cons_arr(i,j,k ,RhoQ3_comp) / cons_arr(i,j,k ,Rho_comp) :
494  Real(0.0);
495  Real qr1 = (rhoqr_comp > -1) ? cons_arr(i,j,k-1,RhoQ3_comp) / cons_arr(i,j,k-1,Rho_comp) :
496  Real(0.0);
497  Real qvface = 0.5 * (qv0 + qv1);
498  Real qcface = 0.5 * (qc0 + qc1);
499  Real qrface = 0.5 * (qr0 + qr1);
500 
501  Real p0 = getPgivenRTh(cons_arr(i, j, k , RhoTheta_comp), qv0) - p0_arr(i,j,k );
502  Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp), qv1) - p0_arr(i,j,k-1);
503  Real pface = 0.5 * (p0 + p1);
504 
505  Real theta0 = cons_arr(i,j,k ,RhoTheta_comp) / cons_arr(i,j,k ,Rho_comp);
506  Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
507  Real thface = 0.5*(theta0 + theta1);
508  Real ql = qcface + qrface;
509  Real thv = thface * (1 + 0.61*qvface - ql);
510 
511  fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k); // p*w
512  fab_arr_stag(i,j,k,6) = qvface * w_fc_arr(i,j,k); // w*qv
513  fab_arr_stag(i,j,k,7) = qcface * w_fc_arr(i,j,k); // w*qc
514  fab_arr_stag(i,j,k,8) = qrface * w_fc_arr(i,j,k); // w*qr
515  fab_arr_stag(i,j,k,9) = thv * w_fc_arr(i,j,k); // w*thv
516  });
517  } // mfi
518  } // use_moisture
519 
520  // Sum in the horizontal plane
521  h_avg_u = sumToLine(u_cc,0,1, domain,zdir);
522  h_avg_v = sumToLine(v_cc,0,1, domain,zdir);
523  h_avg_w = sumToLine(w_fc,0,1,stag_domain,zdir);
524 
525  h_avg_rho = sumToLine(mf_out, 0,1,domain,zdir);
526  h_avg_th = sumToLine(mf_out, 1,1,domain,zdir);
527  h_avg_ksgs = sumToLine(mf_out, 2,1,domain,zdir);
528  h_avg_Kmv = sumToLine(mf_out, 3,1,domain,zdir);
529  h_avg_Khv = sumToLine(mf_out, 4,1,domain,zdir);
530  h_avg_uu = sumToLine(mf_out, 5,1,domain,zdir);
531  h_avg_uv = sumToLine(mf_out, 6,1,domain,zdir);
532  h_avg_vv = sumToLine(mf_out, 7,1,domain,zdir);
533  h_avg_uth = sumToLine(mf_out, 8,1,domain,zdir);
534  h_avg_vth = sumToLine(mf_out, 9,1,domain,zdir);
535  h_avg_thth = sumToLine(mf_out,10,1,domain,zdir);
536  h_avg_uiuiu = sumToLine(mf_out,11,1,domain,zdir);
537  h_avg_uiuiv = sumToLine(mf_out,12,1,domain,zdir);
538  h_avg_p = sumToLine(mf_out,13,1,domain,zdir);
539  h_avg_pu = sumToLine(mf_out,14,1,domain,zdir);
540  h_avg_pv = sumToLine(mf_out,15,1,domain,zdir);
541  h_avg_qv = sumToLine(mf_out,16,1,domain,zdir);
542  h_avg_qc = sumToLine(mf_out,17,1,domain,zdir);
543  h_avg_qr = sumToLine(mf_out,18,1,domain,zdir);
544  h_avg_qi = sumToLine(mf_out,19,1,domain,zdir);
545  h_avg_qs = sumToLine(mf_out,20,1,domain,zdir);
546  h_avg_qg = sumToLine(mf_out,21,1,domain,zdir);
547 
548  h_avg_uw = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
549  h_avg_vw = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
550  h_avg_ww = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
551  h_avg_wth = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
552  h_avg_uiuiw = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
553  h_avg_pw = sumToLine(mf_out_stag,5,1,stag_domain,zdir);
554  h_avg_wqv = sumToLine(mf_out_stag,6,1,stag_domain,zdir);
555  h_avg_wqc = sumToLine(mf_out_stag,7,1,stag_domain,zdir);
556  h_avg_wqr = sumToLine(mf_out_stag,8,1,stag_domain,zdir);
557  h_avg_wthv = sumToLine(mf_out_stag,9,1,stag_domain,zdir);
558 
559  // Divide by the total number of cells we are averaging over
560  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
561  int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
562  for (int k = 0; k < unstag_size; ++k) {
563  h_avg_u[k] /= area_z;
564  h_avg_v[k] /= area_z;
565  h_avg_rho[k] /= area_z;
566  h_avg_ksgs[k] /= area_z;
567  h_avg_Kmv[k] /= area_z;
568  h_avg_Khv[k] /= area_z;
569  h_avg_th[k] /= area_z;
570  h_avg_thth[k] /= area_z;
571  h_avg_uu[k] /= area_z;
572  h_avg_uv[k] /= area_z;
573  h_avg_vv[k] /= area_z;
574  h_avg_uth[k] /= area_z;
575  h_avg_vth[k] /= area_z;
576  h_avg_uiuiu[k] /= area_z;
577  h_avg_uiuiv[k] /= area_z;
578  h_avg_p[k] /= area_z;
579  h_avg_pu[k] /= area_z;
580  h_avg_pv[k] /= area_z;
581  h_avg_qv[k] /= area_z;
582  h_avg_qc[k] /= area_z;
583  h_avg_qr[k] /= area_z;
584  h_avg_qi[k] /= area_z;
585  h_avg_qs[k] /= area_z;
586  h_avg_qg[k] /= area_z;
587  }
588 
589  for (int k = 0; k < unstag_size+1; ++k) { // staggered heights
590  h_avg_w[k] /= area_z;
591  h_avg_uw[k] /= area_z;
592  h_avg_vw[k] /= area_z;
593  h_avg_ww[k] /= area_z;
594  h_avg_wth[k] /= area_z;
595  h_avg_uiuiw[k] /= area_z;
596  h_avg_pw[k] /= area_z;
597  h_avg_wqv[k] /= area_z;
598  h_avg_wqc[k] /= area_z;
599  h_avg_wqr[k] /= area_z;
600  h_avg_wthv[k] /= area_z;
601  }
602 }
const Box zbx
Definition: ERF_SetupDiff.H:9
Here is the call graph for this function:

◆ derive_stress_profiles()

void ERF::derive_stress_profiles ( amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau11,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau12,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau13,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau22,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau23,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau33,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_hfx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q1fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q2fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_diss 
)
480 {
481  int lev = 0;
482 
483  // This will hold the stress tensor components
484  MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
485 
486  MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
487 
488  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
489 
490  for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
491  {
492  const Box& bx = mfi.tilebox();
493  const Array4<Real>& fab_arr = mf_out.array(mfi);
494 
495  const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
496 
497  // NOTE: These are from the last RK stage...
498  const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
499  const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
500  const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
501  const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
502  const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
503  const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
504 
505  // These should be re-calculated during ERF_slow_rhs_post
506  // -- just vertical SFS kinematic heat flux for now
507  //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
508  //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
509  const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
510  const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
511  Array4<const Real>{};
512  const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
513  Array4<const Real>{};
514  const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
515 
516  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
517  {
518  // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
519  fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
520  fab_arr(i, j, k, 1) = ( tau12_arr(i,j ,k) + tau12_arr(i+1,j ,k)
521  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
522  / ( rho_arr(i,j ,k) + rho_arr(i+1,j ,k)
523  + rho_arr(i,j+1,k) + rho_arr(i+1,j+1,k) );
524  fab_arr(i, j, k, 2) = ( tau13_arr(i,j,k ) + tau13_arr(i+1,j,k )
525  + tau13_arr(i,j,k+1) + tau13_arr(i+1,j,k+1) )
526  / ( rho_arr(i,j,k ) + rho_arr(i+1,j,k )
527  + rho_arr(i,j,k+1) + rho_arr(i+1,j,k+1) );
528  fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
529  fab_arr(i, j, k, 4) = ( tau23_arr(i,j,k ) + tau23_arr(i,j+1,k )
530  + tau23_arr(i,j,k+1) + tau23_arr(i,j+1,k+1) )
531  / ( rho_arr(i,j,k ) + rho_arr(i,j+1,k )
532  + rho_arr(i,j,k+1) + rho_arr(i,j+1,k+1) );
533  fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
534  fab_arr(i, j, k, 6) = 0.5 * ( hfx3_arr(i,j,k) + hfx3_arr(i,j,k+1) ) / rho_arr(i,j,k);
535  fab_arr(i, j, k, 7) = (l_use_moist) ? 0.5 * ( q1fx3_arr(i,j,k) + q1fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : 0.0;
536  fab_arr(i, j, k, 8) = (l_use_moist) ? 0.5 * ( q2fx3_arr(i,j,k) + q2fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : 0.0;
537  fab_arr(i, j, k, 9) = diss_arr(i,j,k) / rho_arr(i,j,k);
538  });
539  }
540 
541  int zdir = 2;
542  auto domain = geom[0].Domain();
543 
544  h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
545  h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
546  h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
547  h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
548  h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
549  h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
550  h_avg_hfx3 = sumToLine(mf_out,6,1,domain,zdir);
551  h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
552  h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
553  h_avg_diss = sumToLine(mf_out,9,1,domain,zdir);
554 
555  int ht_size = h_avg_tau11.size();
556 
557  // Divide by the total number of cells we are averaging over
558  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
559  for (int k = 0; k < ht_size; ++k) {
560  h_avg_tau11[k] /= area_z;
561  h_avg_tau12[k] /= area_z;
562  h_avg_tau13[k] /= area_z;
563  h_avg_tau22[k] /= area_z;
564  h_avg_tau23[k] /= area_z;
565  h_avg_tau33[k] /= area_z;
566  h_avg_hfx3[k] /= area_z;
567  h_avg_q1fx3[k] /= area_z;
568  h_avg_q2fx3[k] /= area_z;
569  h_avg_diss[k] /= area_z;
570  }
571 }

◆ derive_stress_profiles_stag()

void ERF::derive_stress_profiles_stag ( amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau11,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau12,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau13,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau22,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau23,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau33,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_hfx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q1fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q2fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_diss 
)
610 {
611  int lev = 0;
612 
613  // This will hold the stress tensor components
614  MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
615 
616  // This will hold Tau13 and Tau23
617  MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 5, 0);
618 
619  MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
620 
621  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
622 
623  for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
624  {
625  const Box& bx = mfi.tilebox();
626  const Array4<Real>& fab_arr = mf_out.array(mfi);
627  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
628 
629  const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
630 
631  // NOTE: These are from the last RK stage...
632  const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
633  const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
634  const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
635  const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
636  const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
637  const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
638 
639  // These should be re-calculated during ERF_slow_rhs_post
640  // -- just vertical SFS kinematic heat flux for now
641  //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
642  //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
643  const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
644  const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
645  Array4<const Real>{};
646  const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
647  Array4<const Real>{};
648  const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
649 
650  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
651  {
652  // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
653  fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
654  fab_arr(i, j, k, 1) = ( tau12_arr(i,j ,k) + tau12_arr(i+1,j ,k)
655  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
656  / ( rho_arr(i,j ,k) + rho_arr(i+1,j ,k)
657  + rho_arr(i,j+1,k) + rho_arr(i+1,j+1,k) );
658  fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
659  fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
660  fab_arr(i, j, k, 9) = diss_arr(i,j,k) / rho_arr(i,j,k);
661  });
662 
663  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
664  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
665  {
666  Real rho_face = 0.5 * (rho_arr(i,j,k-1) + rho_arr(i,j,k));
667  // average from edge to face center
668  fab_arr_stag(i,j,k,0) = 0.5*(tau13_arr(i,j,k) + tau13_arr(i+1,j ,k)) / rho_face;
669  fab_arr_stag(i,j,k,1) = 0.5*(tau23_arr(i,j,k) + tau23_arr(i ,j+1,k)) / rho_face;
670 
671  fab_arr_stag(i,j,k,2) = hfx3_arr(i,j,k) / rho_face;
672  fab_arr_stag(i,j,k,3) = (l_use_moist) ? q1fx3_arr(i,j,k) / rho_face : 0.0;
673  fab_arr_stag(i,j,k,4) = (l_use_moist) ? q2fx3_arr(i,j,k) / rho_face : 0.0;
674  });
675  }
676 
677  int zdir = 2;
678  auto domain = geom[0].Domain();
679  Box stag_domain = domain;
680  stag_domain.convert(IntVect(0,0,1));
681 
682  h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
683  h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
684 // h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
685  h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
686 // h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
687  h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
688 // h_avg_hfx3 = sumToLine(mf_out,6,1,domain,zdir);
689 // h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
690 // h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
691  h_avg_diss = sumToLine(mf_out,9,1,domain,zdir);
692 
693  h_avg_tau13 = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
694  h_avg_tau23 = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
695  h_avg_hfx3 = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
696  h_avg_q1fx3 = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
697  h_avg_q2fx3 = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
698 
699  int ht_size = h_avg_tau11.size(); // _un_staggered
700 
701  // Divide by the total number of cells we are averaging over
702  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
703  for (int k = 0; k < ht_size; ++k) {
704  h_avg_tau11[k] /= area_z;
705  h_avg_tau12[k] /= area_z;
706  h_avg_tau13[k] /= area_z;
707  h_avg_tau22[k] /= area_z;
708  h_avg_tau23[k] /= area_z;
709  h_avg_tau33[k] /= area_z;
710  h_avg_hfx3[k] /= area_z;
711  h_avg_q1fx3[k] /= area_z;
712  h_avg_q2fx3[k] /= area_z;
713  h_avg_diss[k] /= area_z;
714  }
715  // staggered heights
716  h_avg_tau13[ht_size] /= area_z;
717  h_avg_tau23[ht_size] /= area_z;
718  h_avg_hfx3[ht_size] /= area_z;
719  h_avg_q1fx3[ht_size] /= area_z;
720  h_avg_q2fx3[ht_size] /= area_z;
721 }

◆ derive_upwp()

void ERF::derive_upwp ( amrex::Vector< amrex::Real > &  h_havg)

◆ EBFactory()

amrex::EBFArrayBoxFactory const& ERF::EBFactory ( int  lev) const
inlineprivatenoexcept
1628  {
1629  return *(eb[lev]->get_const_factory());
1630  }
amrex::Vector< std::unique_ptr< eb_ > > eb
Definition: ERF.H:1620

Referenced by WriteMyEBSurface().

Here is the caller graph for this function:

◆ erf_enforce_hse()

void ERF::erf_enforce_hse ( int  lev,
amrex::MultiFab &  dens,
amrex::MultiFab &  pres,
amrex::MultiFab &  pi,
amrex::MultiFab &  th,
amrex::MultiFab &  qv,
std::unique_ptr< amrex::MultiFab > &  z_cc 
)

Enforces hydrostatic equilibrium when using terrain.

Parameters
[in]levInteger specifying the current level
[out]densMultiFab storing base state density
[out]presMultiFab storing base state pressure
[out]piMultiFab storing base state Exner function
[in]z_ccPointer to MultiFab storing cell centered z-coordinates
167 {
168  Real l_gravity = solverChoice.gravity;
169  bool l_use_terrain = (solverChoice.mesh_type != MeshType::ConstantDz);
170 
171  const auto geomdata = geom[lev].data();
172  const Real dz = geomdata.CellSize(2);
173 
174  for ( MFIter mfi(dens, TileNoZ()); mfi.isValid(); ++mfi )
175  {
176  // Create a flat box with same horizontal extent but only one cell in vertical
177  const Box& tbz = mfi.nodaltilebox(2);
178  int klo = tbz.smallEnd(2);
179  int khi = tbz.bigEnd(2);
180 
181  // Note we only grow by 1 because that is how big z_cc is.
182  Box b2d = tbz; // Copy constructor
183  b2d.grow(0,1);
184  b2d.grow(1,1);
185  b2d.setRange(2,0);
186 
187  // Intersect this box with the domain
188  Box zdomain = convert(geom[lev].Domain(),tbz.ixType());
189  b2d &= zdomain;
190 
191  // We integrate to the first cell (and below) by using rho in this cell
192  // If gravity == 0 this is constant pressure
193  // If gravity != 0, hence this is a wall, this gives gp0 = dens[0] * gravity
194  // (dens_hse*gravity would also be dens[0]*gravity because we use foextrap for rho at k = -1)
195  // Note ng_pres_hse = 1
196 
197  // We start by assuming pressure on the ground is p_0 (in ERF_Constants.H)
198  // Note that gravity is positive
199 
200  Array4<Real> rho_arr = dens.array(mfi);
201  Array4<Real> pres_arr = pres.array(mfi);
202  Array4<Real> pi_arr = pi.array(mfi);
203  Array4<Real> th_arr = theta.array(mfi);
204  Array4<Real> zcc_arr;
205  if (l_use_terrain) {
206  zcc_arr = z_cc->array(mfi);
207  }
208 
209  const Real rdOcp = solverChoice.rdOcp;
210 
211  ParallelFor(b2d, [=] AMREX_GPU_DEVICE (int i, int j, int)
212  {
213  // Set value at surface from Newton iteration for rho
214  if (klo == 0)
215  {
216  // Physical height of the terrain at cell center
217  Real hz;
218  if (l_use_terrain) {
219  hz = zcc_arr(i,j,klo);
220  } else {
221  hz = 0.5*dz;
222  }
223 
224  pres_arr(i,j,klo) = p_0 - hz * rho_arr(i,j,klo) * l_gravity;
225  pi_arr(i,j,klo) = getExnergivenP(pres_arr(i,j,klo), rdOcp);
226  th_arr(i,j,klo) = getRhoThetagivenP(pres_arr(i,j,klo)) / rho_arr(i,j,klo);
227 
228  //
229  // Set ghost cell with dz and rho at boundary
230  // (We will set the rest of the ghost cells in the boundary condition routine)
231  //
232  pres_arr(i,j,klo-1) = p_0 + hz * rho_arr(i,j,klo) * l_gravity;
233  pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
234  th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
235 
236  } else {
237 
238  // If level > 0 and klo > 0, we need to use the value of pres_arr(i,j,klo-1) which was
239  // filled from FillPatch-ing it.
240  Real dz_loc;
241  if (l_use_terrain) {
242  dz_loc = (zcc_arr(i,j,klo) - zcc_arr(i,j,klo-1));
243  } else {
244  dz_loc = dz;
245  }
246 
247  Real dens_interp = 0.5*(rho_arr(i,j,klo) + rho_arr(i,j,klo-1));
248  pres_arr(i,j,klo) = pres_arr(i,j,klo-1) - dz_loc * dens_interp * l_gravity;
249 
250  pi_arr(i,j,klo ) = getExnergivenP(pres_arr(i,j,klo ), rdOcp);
251  th_arr(i,j,klo ) = getRhoThetagivenP(pres_arr(i,j,klo )) / rho_arr(i,j,klo );
252 
253  pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
254  th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
255  }
256 
257  Real dens_interp;
258  if (l_use_terrain) {
259  for (int k = klo+1; k <= khi; k++) {
260  Real dz_loc = (zcc_arr(i,j,k) - zcc_arr(i,j,k-1));
261  dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
262  pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz_loc * dens_interp * l_gravity;
263  pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
264  th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
265  }
266  } else {
267  for (int k = klo+1; k <= khi; k++) {
268  dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
269  pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz * dens_interp * l_gravity;
270  pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
271  th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
272  }
273  }
274  });
275 
276  } // mfi
277 
278  dens.FillBoundary(geom[lev].periodicity());
279  pres.FillBoundary(geom[lev].periodicity());
280  pi.FillBoundary(geom[lev].periodicity());
281  theta.FillBoundary(geom[lev].periodicity());
282  qv.FillBoundary(geom[lev].periodicity());
283 }
constexpr amrex::Real p_0
Definition: ERF_Constants.H:18
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getRhoThetagivenP(const amrex::Real p, const amrex::Real qv=0.0)
Definition: ERF_EOS.H:172
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getExnergivenP(const amrex::Real P, const amrex::Real rdOcp)
Definition: ERF_EOS.H:141
@ pres
Definition: ERF_Kessler.H:25
real(c_double), parameter, private pi
Definition: ERF_module_mp_morr_two_moment.F90:100
amrex::Real rdOcp
Definition: ERF_DataStruct.H:982
amrex::Real gravity
Definition: ERF_DataStruct.H:980
Here is the call graph for this function:

◆ ERF_shared()

void ERF::ERF_shared ( )
144 {
145  if (ParallelDescriptor::IOProcessor()) {
146  const char* erf_hash = buildInfoGetGitHash(1);
147  const char* amrex_hash = buildInfoGetGitHash(2);
148  const char* buildgithash = buildInfoGetBuildGitHash();
149  const char* buildgitname = buildInfoGetBuildGitName();
150 
151  if (strlen(erf_hash) > 0) {
152  Print() << "\n"
153  << "ERF git hash: " << erf_hash << "\n";
154  }
155  if (strlen(amrex_hash) > 0) {
156  Print() << "AMReX git hash: " << amrex_hash << "\n";
157  }
158  if (strlen(buildgithash) > 0) {
159  Print() << buildgitname << " git hash: " << buildgithash << "\n";
160  }
161 
162  Print() << "\n";
163  }
164 
165  int nlevs_max = max_level + 1;
166 
167 #ifdef ERF_USE_WINDFARM
168  Nturb.resize(nlevs_max);
169  vars_windfarm.resize(nlevs_max);
170  SMark.resize(nlevs_max);
171 #endif
172 
173  qheating_rates.resize(nlevs_max);
174  rad_fluxes.resize(nlevs_max);
175  sw_lw_fluxes.resize(nlevs_max);
176  solar_zenith.resize(nlevs_max);
177 
178  // NOTE: size lsm before readparams (chooses the model at all levels)
179  lsm.ReSize(nlevs_max);
180  lsm_data.resize(nlevs_max);
181  lsm_flux.resize(nlevs_max);
182 
183  // NOTE: size canopy model before readparams (if file exists, we construct)
184  m_forest_drag.resize(nlevs_max);
185  for (int lev = 0; lev <= max_level; ++lev) { m_forest_drag[lev] = nullptr;}
186 
187  ReadParameters();
188  initializeMicrophysics(nlevs_max);
189 
190 #ifdef ERF_USE_WINDFARM
191  initializeWindFarm(nlevs_max);
192 #endif
193 
194 #ifdef ERF_USE_SHOC
195  shoc_interface.resize(nlevs_max);
196  if (solverChoice.use_shoc) {
197  for (int lev = 0; lev <= max_level; ++lev) {
198  shoc_interface[lev] = std::make_unique<SHOCInterface>(lev, solverChoice);
199  }
200  }
201 #endif
202 
203  rad.resize(nlevs_max);
204  for (int lev = 0; lev <= max_level; ++lev) {
205  if (solverChoice.rad_type == RadiationType::RRTMGP) {
206 #ifdef ERF_USE_RRTMGP
207  rad[lev] = std::make_unique<Radiation>(lev, solverChoice);
208  // pass radiation datalog frequency to model - RRTMGP needs to know when to save data for profiles
209  rad[lev]->setDataLogFrequency(rad_datalog_int);
210 #endif
211  } else if (solverChoice.rad_type != RadiationType::None) {
212  Abort("Don't know this radiation model!");
213  }
214  }
215 
216  const std::string& pv3d_1 = "plot_vars_1" ; setPlotVariables(pv3d_1,plot3d_var_names_1);
217  const std::string& pv3d_2 = "plot_vars_2" ; setPlotVariables(pv3d_2,plot3d_var_names_2);
218  const std::string& pv2d_1 = "plot2d_vars_1"; setPlotVariables2D(pv2d_1,plot2d_var_names_1);
219  const std::string& pv2d_2 = "plot2d_vars_2"; setPlotVariables2D(pv2d_2,plot2d_var_names_2);
220 
221  // This is only used when we have mesh_type == MeshType::StretchedDz
222  stretched_dz_h.resize(nlevs_max);
223  stretched_dz_d.resize(nlevs_max);
224 
225  // Initialize staggered vertical levels for grid stretching or terrain, and
226  // to simplify Rayleigh damping layer calculations.
227  zlevels_stag.resize(max_level+1);
231  geom,
232  refRatio(),
235  solverChoice.dz0);
236 
237  if (SolverChoice::mesh_type == MeshType::StretchedDz ||
238  SolverChoice::mesh_type == MeshType::VariableDz) {
239  int nz = geom[0].Domain().length(2) + 1; // staggered
240  if (std::fabs(zlevels_stag[0][nz-1]-geom[0].ProbHi(2)) > 1.0e-4) {
241  Print() << "Note: prob_hi[2]=" << geom[0].ProbHi(2)
242  << " does not match highest requested z level " << zlevels_stag[0][nz-1]
243  << std::endl;
244  }
245  if (std::fabs(zlevels_stag[0][0]-geom[0].ProbLo(2)) > 1.0e-4) {
246  Print() << "Note: prob_lo[2]=" << geom[0].ProbLo(2)
247  << " does not match lowest requested level " << zlevels_stag[0][0]
248  << std::endl;
249  }
250 
251  // Redefine the problem domain here?
252  }
253 
254  // Get lo/hi indices for massflux calc
256  if (solverChoice.mesh_type == MeshType::ConstantDz) {
257  const Real massflux_zlo = solverChoice.const_massflux_layer_lo - geom[0].ProbLo(2);
258  const Real massflux_zhi = solverChoice.const_massflux_layer_hi - geom[0].ProbLo(2);
259  const Real dz = geom[0].CellSize(2);
260  if (massflux_zlo == -1e34) {
261  solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
262  } else {
263  solverChoice.massflux_klo = static_cast<int>(std::ceil(massflux_zlo / dz - 0.5));
264  }
265  if (massflux_zhi == 1e34) {
266  solverChoice.massflux_khi = geom[0].Domain().bigEnd(2);
267  } else {
268  solverChoice.massflux_khi = static_cast<int>(std::floor(massflux_zhi / dz - 0.5));
269  }
270  } else if (solverChoice.mesh_type == MeshType::StretchedDz) {
271  const Real massflux_zlo = solverChoice.const_massflux_layer_lo;
272  const Real massflux_zhi = solverChoice.const_massflux_layer_hi;
273  solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
274  solverChoice.massflux_khi = geom[0].Domain().bigEnd(2) + 1;
275  for (int k=0; k <= geom[0].Domain().bigEnd(2)+1; ++k) {
276  if (zlevels_stag[0][k] <= massflux_zlo) solverChoice.massflux_klo = k;
277  if (zlevels_stag[0][k] <= massflux_zhi) solverChoice.massflux_khi = k;
278  }
279  } else { // solverChoice.mesh_type == MeshType::VariableDz
280  Error("Const massflux with variable dz not supported -- planar averages are on k rather than constant-z planes");
281  }
282 
283  Print() << "Constant mass flux based on k in ["
284  << solverChoice.massflux_klo << ", " << solverChoice.massflux_khi << "]" << std::endl;
285  }
286 
287  prob = amrex_probinit(geom[0].ProbLo(),geom[0].ProbHi());
288 
289  // Geometry on all levels has been defined already.
290 
291  // No valid BoxArray and DistributionMapping have been defined.
292  // But the arrays for them have been resized.
293 
294  t_new.resize(nlevs_max, 0.0);
295  t_old.resize(nlevs_max, -1.e100);
296  dt.resize(nlevs_max, std::min(1.e100,dt_max_initial));
297  dt_mri_ratio.resize(nlevs_max, 1);
298 
299  vars_new.resize(nlevs_max);
300  vars_old.resize(nlevs_max);
301  gradp.resize(nlevs_max);
302 
303  // We resize this regardless in order to pass it without error
304  pp_inc.resize(nlevs_max);
305 
306  // Used in the fast substepping only
307  lagged_delta_rt.resize(nlevs_max);
308  avg_xmom.resize(nlevs_max);
309  avg_ymom.resize(nlevs_max);
310  avg_zmom.resize(nlevs_max);
311 
312  rU_new.resize(nlevs_max);
313  rV_new.resize(nlevs_max);
314  rW_new.resize(nlevs_max);
315 
316  rU_old.resize(nlevs_max);
317  rV_old.resize(nlevs_max);
318  rW_old.resize(nlevs_max);
319 
320  // xmom_crse_rhs.resize(nlevs_max);
321  // ymom_crse_rhs.resize(nlevs_max);
322  zmom_crse_rhs.resize(nlevs_max);
323 
324  for (int lev = 0; lev < nlevs_max; ++lev) {
325  vars_new[lev].resize(Vars::NumTypes);
326  vars_old[lev].resize(Vars::NumTypes);
327  gradp[lev].resize(AMREX_SPACEDIM);
328  }
329 
330  // Time integrator
331  mri_integrator_mem.resize(nlevs_max);
332 
333  // Physical boundary conditions
334  physbcs_cons.resize(nlevs_max);
335  physbcs_u.resize(nlevs_max);
336  physbcs_v.resize(nlevs_max);
337  physbcs_w.resize(nlevs_max);
338  physbcs_base.resize(nlevs_max);
339 
340  // Planes to hold Dirichlet values at boundaries
341  xvel_bc_data.resize(nlevs_max);
342  yvel_bc_data.resize(nlevs_max);
343  zvel_bc_data.resize(nlevs_max);
344  th_bc_data.resize(nlevs_max);
345 
346  advflux_reg.resize(nlevs_max);
347 
348  // Stresses
349  Tau.resize(nlevs_max);
350  Tau_corr.resize(nlevs_max);
351  SFS_hfx1_lev.resize(nlevs_max); SFS_hfx2_lev.resize(nlevs_max); SFS_hfx3_lev.resize(nlevs_max);
352  SFS_diss_lev.resize(nlevs_max);
353  SFS_q1fx1_lev.resize(nlevs_max); SFS_q1fx2_lev.resize(nlevs_max); SFS_q1fx3_lev.resize(nlevs_max);
354  SFS_q2fx3_lev.resize(nlevs_max);
355  eddyDiffs_lev.resize(nlevs_max);
356  SmnSmn_lev.resize(nlevs_max);
357 
358  // Sea surface temps
359  sst_lev.resize(nlevs_max);
360  tsk_lev.resize(nlevs_max);
361  lmask_lev.resize(nlevs_max);
362 
363  // Land and soil grid type and urban fractions
364  land_type_lev.resize(nlevs_max);
365  soil_type_lev.resize(nlevs_max);
366  urb_frac_lev.resize(nlevs_max);
367 
368  // Metric terms
369  z_phys_nd.resize(nlevs_max);
370  z_phys_cc.resize(nlevs_max);
371  detJ_cc.resize(nlevs_max);
372  ax.resize(nlevs_max);
373  ay.resize(nlevs_max);
374  az.resize(nlevs_max);
375 
376  z_phys_nd_new.resize(nlevs_max);
377  detJ_cc_new.resize(nlevs_max);
378 
379  z_phys_nd_src.resize(nlevs_max);
380  z_phys_cc_src.resize(nlevs_max);
381  detJ_cc_src.resize(nlevs_max);
382  ax_src.resize(nlevs_max);
383  ay_src.resize(nlevs_max);
384  az_src.resize(nlevs_max);
385 
386  z_t_rk.resize(nlevs_max);
387 
388  terrain_blanking.resize(nlevs_max);
389 
390  // Wall distance
391  walldist.resize(nlevs_max);
392 
393  // BoxArrays to make MultiFabs needed to convert WRFBdy data
394  ba1d.resize(nlevs_max);
395  ba2d.resize(nlevs_max);
396 
397  // MultiFabs needed to convert WRFBdy data
398  mf_PSFC.resize(nlevs_max);
399 
400  // Map factors
401  mapfac.resize(nlevs_max);
402 
403  // Thin immersed body
404  xflux_imask.resize(nlevs_max);
405  yflux_imask.resize(nlevs_max);
406  zflux_imask.resize(nlevs_max);
407  //overset_imask.resize(nlevs_max);
408  thin_xforce.resize(nlevs_max);
409  thin_yforce.resize(nlevs_max);
410  thin_zforce.resize(nlevs_max);
411 
412  // Base state
413  base_state.resize(nlevs_max);
414  base_state_new.resize(nlevs_max);
415 
416  // Wave coupling data
417  Hwave.resize(nlevs_max);
418  Lwave.resize(nlevs_max);
419  for (int lev = 0; lev < max_level; ++lev)
420  {
421  Hwave[lev] = nullptr;
422  Lwave[lev] = nullptr;
423  }
424  Hwave_onegrid.resize(nlevs_max);
425  Lwave_onegrid.resize(nlevs_max);
426  for (int lev = 0; lev < max_level; ++lev)
427  {
428  Hwave_onegrid[lev] = nullptr;
429  Lwave_onegrid[lev] = nullptr;
430  }
431 
432  // Theta prim for MOST
433  Theta_prim.resize(nlevs_max);
434 
435  // Qv prim for MOST
436  Qv_prim.resize(nlevs_max);
437 
438  // Qr prim for MOST
439  Qr_prim.resize(nlevs_max);
440 
441  // Time averaged velocity field
442  vel_t_avg.resize(nlevs_max);
443  t_avg_cnt.resize(nlevs_max);
444 
445  // Size lat long arrays and default to null pointers
446  lat_m.resize(nlevs_max);
447  lon_m.resize(nlevs_max);
448  for (int lev = 0; lev < max_level; ++lev) {
449  lat_m[lev] = nullptr;
450  lon_m[lev] = nullptr;
451  }
452 
453  // Variable coriolis
454  sinPhi_m.resize(nlevs_max);
455  cosPhi_m.resize(nlevs_max);
456  for (int lev = 0; lev < max_level; ++lev) {
457  sinPhi_m[lev] = nullptr;
458  cosPhi_m[lev] = nullptr;
459  }
460 
461  // Initialize tagging criteria for mesh refinement
463 
464  for (int lev = 0; lev < max_level; ++lev)
465  {
466  Print() << "Refinement ratio at level " << lev+1 << " set to be " <<
467  ref_ratio[lev][0] << " " << ref_ratio[lev][1] << " " << ref_ratio[lev][2] << std::endl;
468  }
469 
470  // We will create each of these in MakeNewLevelFromScratch
471  eb.resize(max_level+1);
472  for (int lev = 0; lev < max_level + 1; lev++){
473  eb[lev] = std::make_unique<eb_>();
474  }
475 
476  //
477  // Construct the EB data structures and store in a separate class
478  //
479  // This is needed before initializing level MultiFabs
480  if ( solverChoice.terrain_type == TerrainType::EB ||
481  solverChoice.terrain_type == TerrainType::ImmersedForcing)
482  {
483  std::string geometry ="terrain";
484  ParmParse pp("eb2");
485  pp.queryAdd("geometry", geometry);
486 
487  int ngrow_for_eb = 4; // This is the default in amrex but we need to explicitly pass it here since
488  // we want to also pass the build_coarse_level_by_coarsening argument
489  if (geometry == "terrain") {
490  Box terrain_bx(surroundingNodes(geom[max_level].Domain())); terrain_bx.grow(3);
491  FArrayBox terrain_fab(makeSlab(terrain_bx,2,0),1);
492  Real dummy_time = 0.0;
493  prob->init_terrain_surface(geom[max_level], terrain_fab, dummy_time);
494  TerrainIF implicit_fun(terrain_fab, geom[max_level], stretched_dz_d[max_level]);
495  auto gshop = EB2::makeShop(implicit_fun);
496  amrex::EB2::Build(gshop, this->Geom(), ngrow_for_eb);
497  } else if (geometry == "box") {
498  RealArray box_lo{0.0, 0.0, 0.0};
499  RealArray box_hi{0.0, 0.0, 0.0};
500  pp.query("box_lo", box_lo);
501  pp.query("box_hi", box_hi);
502  EB2::BoxIF implicit_fun(box_lo, box_hi, false);
503  auto gshop = EB2::makeShop(implicit_fun);
504  amrex::EB2::Build(gshop, this->Geom(), ngrow_for_eb);
505  } else if (geometry == "sphere") {
506  auto ProbLoArr = geom[max_level].ProbLoArray();
507  auto ProbHiArr = geom[max_level].ProbHiArray();
508  const Real xcen = 0.5 * (ProbLoArr[0] + ProbHiArr[0]);
509  const Real ycen = 0.5 * (ProbLoArr[1] + ProbHiArr[1]);
510  RealArray sphere_center = {xcen, ycen, 0.0};
511  EB2::SphereIF implicit_fun(0.5, sphere_center, false);
512  auto gshop = EB2::makeShop(implicit_fun);
513  amrex::EB2::Build(gshop, this->Geom(), ngrow_for_eb);
514  }
515  }
516 }
void init_zlevels(Vector< Vector< Real >> &zlevels_stag, Vector< Vector< Real >> &stretched_dz_h, Vector< Gpu::DeviceVector< Real >> &stretched_dz_d, Vector< Geometry > const &geom, Vector< IntVect > const &ref_ratio, const Real grid_stretching_ratio, const Real zsurf, const Real dz0)
Definition: ERF_InitZLevels.cpp:11
std::unique_ptr< ProblemBase > amrex_probinit(const amrex_real *problo, const amrex_real *probhi) AMREX_ATTRIBUTE_WEAK
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave_onegrid
Definition: ERF.H:968
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_yforce
Definition: ERF.H:1001
void setPlotVariables(const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
Definition: ERF_Plotfile.cpp:24
amrex::Vector< amrex::BoxArray > ba2d
Definition: ERF.H:1245
amrex::Vector< amrex::Vector< amrex::MultiFab > > gradp
Definition: ERF.H:815
void ReadParameters()
Definition: ERF.cpp:2067
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_PSFC
Definition: ERF.H:1250
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_src
Definition: ERF.H:938
amrex::Vector< amrex::MultiFab > base_state_new
Definition: ERF.H:963
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az
Definition: ERF.H:936
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > lmask_lev
Definition: ERF.H:913
amrex::Vector< std::unique_ptr< amrex::MultiFab > > terrain_blanking
Definition: ERF.H:951
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_new
Definition: ERF.H:945
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_zforce
Definition: ERF.H:1002
amrex::Vector< std::string > plot3d_var_names_2
Definition: ERF.H:1096
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > sst_lev
Definition: ERF.H:911
amrex::Vector< std::string > plot2d_var_names_1
Definition: ERF.H:1097
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_xforce
Definition: ERF.H:1000
void setPlotVariables2D(const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
Definition: ERF_Plotfile.cpp:186
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > th_bc_data
Definition: ERF.H:770
amrex::Vector< amrex::Real > t_old
Definition: ERF.H:804
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_t_rk
Definition: ERF.H:948
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave_onegrid
Definition: ERF.H:969
amrex::Vector< std::unique_ptr< ForestDrag > > m_forest_drag
Definition: ERF.H:1330
amrex::Vector< amrex::BoxArray > ba1d
Definition: ERF.H:1244
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > xvel_bc_data
Definition: ERF.H:767
int rad_datalog_int
Definition: ERF.H:894
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_src
Definition: ERF.H:940
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay_src
Definition: ERF.H:942
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > yflux_imask
Definition: ERF.H:995
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_flux
Definition: ERF.H:876
amrex::Vector< std::string > plot3d_var_names_1
Definition: ERF.H:1095
void refinement_criteria_setup()
Definition: ERF_Tagging.cpp:237
amrex::Vector< std::string > plot2d_var_names_2
Definition: ERF.H:1098
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau_corr
Definition: ERF.H:906
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sinPhi_m
Definition: ERF.H:758
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax_src
Definition: ERF.H:941
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > urb_frac_lev
Definition: ERF.H:918
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc_src
Definition: ERF.H:939
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > soil_type_lev
Definition: ERF.H:917
amrex::Vector< amrex::Vector< amrex::Real > > zlevels_stag
Definition: ERF.H:927
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_data
Definition: ERF.H:874
amrex::Vector< amrex::Vector< amrex::Real > > stretched_dz_h
Definition: ERF.H:959
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az_src
Definition: ERF.H:943
static amrex::Real dt_max_initial
Definition: ERF.H:1047
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave
Definition: ERF.H:967
amrex::Vector< std::unique_ptr< amrex::MultiFab > > cosPhi_m
Definition: ERF.H:758
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > land_type_lev
Definition: ERF.H:916
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > zflux_imask
Definition: ERF.H:996
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > zvel_bc_data
Definition: ERF.H:769
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_new
Definition: ERF.H:946
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > yvel_bc_data
Definition: ERF.H:768
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave
Definition: ERF.H:966
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > xflux_imask
Definition: ERF.H:994
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > tsk_lev
Definition: ERF.H:912
void initializeMicrophysics(const int &)
Definition: ERF.cpp:1834
void ReSize(const int &nlev)
Definition: ERF_LandSurface.H:24
Definition: ERF_EBIFTerrain.H:14
const char * buildInfoGetGitHash(int i)
amrex::Real dz0
Definition: ERF_DataStruct.H:987
amrex::Real const_massflux_layer_lo
Definition: ERF_DataStruct.H:1064
amrex::Real const_massflux_v
Definition: ERF_DataStruct.H:1062
int massflux_klo
Definition: ERF_DataStruct.H:1066
amrex::Real grid_stretching_ratio
Definition: ERF_DataStruct.H:985
amrex::Real const_massflux_u
Definition: ERF_DataStruct.H:1061
amrex::Real zsurf
Definition: ERF_DataStruct.H:986
amrex::Real const_massflux_layer_hi
Definition: ERF_DataStruct.H:1065
int massflux_khi
Definition: ERF_DataStruct.H:1067
Here is the call graph for this function:

◆ ErrorEst()

void ERF::ErrorEst ( int  lev,
amrex::TagBoxArray &  tags,
amrex::Real  time,
int  ngrow 
)
override

Function to tag cells for refinement – this overrides the pure virtual function in AmrCore

Parameters
[in]levclevel of refinement at which we tag cells (0 is coarsest level)
[out]tagsarray of tagged cells
[in]timecurrent time
21 {
22  const int clearval = TagBox::CLEAR;
23  const int tagval = TagBox::SET;
24 
25 #ifdef ERF_USE_NETCDF
26  if (solverChoice.init_type == InitType::WRFInput) {
27  int ratio;
28  Box subdomain;
29 
30  if (!nc_init_file[levc+1].empty())
31  {
32  Real levc_start_time = read_start_time_from_wrfinput(levc , nc_init_file[levc ][0]);
33  Real levf_start_time = read_start_time_from_wrfinput(levc+1, nc_init_file[levc+1][0]);
34  amrex::Print() << " WRFInput time at level " << levc << " is " << levc_start_time << std::endl;
35  amrex::Print() << " WRFInput start_time at level " << levc+1 << " is " << levf_start_time << std::endl;
36 
37  if ( levf_start_time <= (levc_start_time + t_new[levc]) ) {
38  amrex::Print() << " WRFInput file to read: " << nc_init_file[levc+1][0] << std::endl;
39  subdomain = read_subdomain_from_wrfinput(levc, nc_init_file[levc+1][0], ratio);
40  amrex::Print() << " WRFInput subdomain at level " << levc+1 << " is " << subdomain << std::endl;
41 
42  if ( (ratio != ref_ratio[levc][0]) || (ratio != ref_ratio[levc][1]) ) {
43  amrex::Print() << "File " << nc_init_file[levc+1][0] << " has refinement ratio = " << ratio << std::endl;
44  amrex::Print() << "The inputs file has refinement ratio = " << ref_ratio[levc] << std::endl;
45  amrex::Abort("These must be the same -- please edit your inputs file and try again.");
46  }
47 
48  if ( (ref_ratio[levc][2]) != 1) {
49  amrex::Abort("The ref_ratio specified in the inputs file must have 1 in the z direction; please use ref_ratio_vect rather than ref_ratio");
50  }
51 
52  subdomain.coarsen(IntVect(ratio,ratio,1));
53 
54  // We assume there is only one subdomain at levc; otherwise we don't know
55  // which one is the parent of the fine region we are trying to create
56  AMREX_ALWAYS_ASSERT(subdomains[levc].size() == 1);
57 
58  Box coarser_level(subdomains[levc][0].minimalBox());
59  subdomain.shift(coarser_level.smallEnd());
60 
61  if (verbose > 0) {
62  amrex::Print() << " Crse subdomain to be tagged is" << subdomain << std::endl;
63  }
64 
65  Box new_fine(subdomain); new_fine.refine(IntVect(ratio,ratio,1));
66  num_boxes_at_level[levc+1] = 1;
67  boxes_at_level[levc+1].push_back(new_fine);
68 
69  for (MFIter mfi(tags); mfi.isValid(); ++mfi) {
70  auto tag_arr = tags.array(mfi); // Get device-accessible array
71 
72  Box bx = mfi.validbox(); bx &= subdomain;
73 
74  if (!bx.isEmpty()) {
75  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
76  tag_arr(i,j,k) = TagBox::SET;
77  });
78  }
79  }
80 
81  } else { // start_Time
82 
83  amrex::Print() << " Not creating level " << levc+1 << " at this time \n " << std::endl;
84  return;
85  }
86  } // file not empty
87 
88  return;
89  }
90 #endif
91 
92  //
93  // Make sure the ghost cells of the level we are tagging at are filled
94  // in case we take differences that require them
95  // NOTE: We are Fillpatching only the cell-centered variables here
96  //
97  MultiFab& S_new = vars_new[levc][Vars::cons];
98  MultiFab& U_new = vars_new[levc][Vars::xvel];
99  MultiFab& V_new = vars_new[levc][Vars::yvel];
100  MultiFab& W_new = vars_new[levc][Vars::zvel];
101  //
102  if (levc == 0) {
103  FillPatchCrseLevel(levc, time, {&S_new, &U_new, &V_new, &W_new});
104  } else {
105  FillPatchFineLevel(levc, time, {&S_new, &U_new, &V_new, &W_new},
106  {&S_new, &rU_new[levc], &rV_new[levc], &rW_new[levc]},
107  base_state[levc], base_state[levc],
108  false, true);
109  }
110 
111  for (int j=0; j < ref_tags.size(); ++j)
112  {
113  //
114  // This mf must have ghost cells because we may take differences between adjacent values
115  //
116  std::unique_ptr<MultiFab> mf = std::make_unique<MultiFab>(grids[levc], dmap[levc], 1, 1);
117 
118  // This allows dynamic refinement based on the value of the density
119  if (ref_tags[j].Field() == "density")
120  {
121  MultiFab::Copy(*mf,vars_new[levc][Vars::cons],Rho_comp,0,1,1);
122 
123  // This allows dynamic refinement based on the value of qv
124  } else if ( ref_tags[j].Field() == "qv" ) {
125  MultiFab::Copy( *mf, vars_new[levc][Vars::cons], RhoQ1_comp, 0, 1, 1);
126  MultiFab::Divide(*mf, vars_new[levc][Vars::cons], Rho_comp, 0, 1, 1);
127 
128 
129  // This allows dynamic refinement based on the value of qc
130  } else if (ref_tags[j].Field() == "qc" ) {
131  MultiFab::Copy( *mf, vars_new[levc][Vars::cons], RhoQ2_comp, 0, 1, 1);
132  MultiFab::Divide(*mf, vars_new[levc][Vars::cons], Rho_comp, 0, 1, 1);
133 
134  // This allows dynamic refinement based on the value of the z-component of vorticity
135  } else if (ref_tags[j].Field() == "vorticity" ) {
136  Vector<MultiFab> mf_cc_vel(1);
137  mf_cc_vel[0].define(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(1,1,1));
138  average_face_to_cellcenter(mf_cc_vel[0],0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new});
139 
140  // Impose bc's at domain boundaries at all levels
141  FillBdyCCVels(mf_cc_vel,levc);
142 
143  mf->setVal(0.);
144 
145  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
146  {
147  const Box& bx = mfi.tilebox();
148  auto& dfab = (*mf)[mfi];
149  auto& sfab = mf_cc_vel[0][mfi];
150  derived::erf_dervortz(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
151  }
152 
153  // This allows dynamic refinement based on the value of the scalar/theta
154  } else if ( (ref_tags[j].Field() == "scalar" ) ||
155  (ref_tags[j].Field() == "theta" ) )
156  {
157  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
158  {
159  const Box& bx = mfi.growntilebox();
160  auto& dfab = (*mf)[mfi];
161  auto& sfab = vars_new[levc][Vars::cons][mfi];
162  if (ref_tags[j].Field() == "scalar") {
163  derived::erf_derscalar(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
164  } else if (ref_tags[j].Field() == "theta") {
165  derived::erf_dertheta(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
166  }
167  } // mfi
168  // This allows dynamic refinement based on the value of the density
169  } else if ( (SolverChoice::terrain_type == TerrainType::ImmersedForcing) &&
170  (ref_tags[j].Field() == "terrain_blanking") )
171  {
172  MultiFab::Copy(*mf,*terrain_blanking[levc],0,0,1,1);
173  } else if (ref_tags[j].Field() == "velmag") {
174  mf->setVal(0.0);
175  ParmParse pp(pp_prefix);
176  Vector<std::string> refinement_indicators;
177  pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
178  Real velmag_threshold = 1e10;
179  for (int i=0; i<refinement_indicators.size(); ++i)
180  {
181  if(refinement_indicators[i]=="hurricane_tracker"){
182  std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
183  ParmParse ppr(ref_prefix);
184  ppr.get("value_greater",velmag_threshold);
185  break;
186  }
187  }
188  HurricaneTracker(levc, U_new, V_new, W_new, velmag_threshold, false, &tags);
189 #ifdef ERF_USE_PARTICLES
190  } else {
191  //
192  // This allows dynamic refinement based on the number of particles per cell
193  //
194  // Note that we must count all the particles in levels both at and above the current,
195  // since otherwise, e.g., if the particles are all at level 1, counting particles at
196  // level 0 will not trigger refinement when regridding so level 1 will disappear,
197  // then come back at the next regridding
198  //
199  const auto& particles_namelist( particleData.getNames() );
200  mf->setVal(0.0);
201  for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++)
202  {
203  std::string tmp_string(particles_namelist[i]+"_count");
204  IntVect rr = IntVect::TheUnitVector();
205  if (ref_tags[j].Field() == tmp_string) {
206  for (int lev = levc; lev <= finest_level; lev++)
207  {
208  MultiFab temp_dat(grids[lev], dmap[lev], 1, 0); temp_dat.setVal(0);
209  particleData[particles_namelist[i]]->IncrementWithTotal(temp_dat, lev);
210 
211  MultiFab temp_dat_crse(grids[levc], dmap[levc], 1, 0); temp_dat_crse.setVal(0);
212 
213  if (lev == levc) {
214  MultiFab::Copy(*mf, temp_dat, 0, 0, 1, 0);
215  } else {
216  for (int d = 0; d < AMREX_SPACEDIM; d++) {
217  rr[d] *= ref_ratio[levc][d];
218  }
219  average_down(temp_dat, temp_dat_crse, 0, 1, rr);
220  MultiFab::Add(*mf, temp_dat_crse, 0, 0, 1, 0);
221  }
222  }
223  }
224  }
225 #endif
226  }
227 
228  ref_tags[j](tags,mf.get(),clearval,tagval,time,levc,geom[levc]);
229  } // loop over j
230 }
amrex::Vector< amrex::Vector< amrex::Box > > boxes_at_level
Definition: ERF.H:797
void FillBdyCCVels(amrex::Vector< amrex::MultiFab > &mf_cc_vel, int levc=0)
Definition: ERF_FillBdyCCVels.cpp:11
void HurricaneTracker(int lev, const amrex::MultiFab &U_new, const amrex::MultiFab &V_new, const amrex::MultiFab &W_new, const amrex::Real velmag_threshold, const bool is_track_io, amrex::TagBoxArray *tags=nullptr)
Definition: ERF_Tagging.cpp:465
void FillPatchCrseLevel(int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)
Definition: ERF_FillPatch.cpp:282
static amrex::Vector< amrex::Vector< std::string > > nc_init_file
Definition: ERF.H:1218
amrex::Vector< amrex::Vector< amrex::BoxArray > > subdomains
Definition: ERF.H:1337
static amrex::Vector< amrex::AMRErrorTag > ref_tags
Definition: ERF.H:1335
amrex::Vector< int > num_boxes_at_level
Definition: ERF.H:795
void erf_derscalar(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:165
void erf_dervortz(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:256
void erf_dertheta(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:144
real(c_double), private rr
Definition: ERF_module_mp_morr_two_moment.F90:224
Here is the call graph for this function:

◆ estTimeStep()

Real ERF::estTimeStep ( int  level,
long &  dt_fast_ratio 
) const

Function that calls estTimeStep for each level

Parameters
[in]levellevel of refinement (coarsest level i 0)
[out]dt_fast_ratioratio of slow to fast time step
55 {
56  BL_PROFILE("ERF::estTimeStep()");
57 
58  Real estdt_comp = 1.e20;
59  Real estdt_lowM = 1.e20;
60 
61  // We intentionally use the level 0 domain to compute whether to use this direction in the dt calculation
62  const int nxc = geom[0].Domain().length(0);
63  const int nyc = geom[0].Domain().length(1);
64 
65  auto const dxinv = geom[level].InvCellSizeArray();
66  auto const dzinv = 1.0 / dz_min[level];
67 
68  MultiFab const& S_new = vars_new[level][Vars::cons];
69 
70  MultiFab ccvel(grids[level],dmap[level],3,0);
71 
72  average_face_to_cellcenter(ccvel,0,
73  Array<const MultiFab*,3>{&vars_new[level][Vars::xvel],
74  &vars_new[level][Vars::yvel],
75  &vars_new[level][Vars::zvel]});
76 
77  bool l_substepping = (solverChoice.substepping_type[level] == SubsteppingType::Implicit);
78  int l_anelastic = solverChoice.anelastic[level];
79 
80  Real estdt_comp_inv;
81 
82  if (l_substepping && (nxc==1) && (nyc==1)) {
83  // SCM -- should not depend on dx or dy; force minimum number of substeps
84  estdt_comp_inv = std::numeric_limits<Real>::min();
85  }
86  else if (solverChoice.terrain_type == TerrainType::EB)
87  {
88  const eb_& eb_lev = get_eb(level);
89  const MultiFab& detJ = (eb_lev.get_const_factory())->getVolFrac();
90 
91  estdt_comp_inv = ReduceMax(S_new, ccvel, detJ, 0,
92  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
93  Array4<Real const> const& s,
94  Array4<Real const> const& u,
95  Array4<Real const> const& vf) -> Real
96  {
97  Real new_comp_dt = -1.e100;
98  amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
99  {
100  if (vf(i,j,k) > 0.)
101  {
102  const Real rho = s(i, j, k, Rho_comp);
103  const Real rhotheta = s(i, j, k, RhoTheta_comp);
104 
105  // NOTE: even when moisture is present,
106  // we only use the partial pressure of the dry air
107  // to compute the soundspeed
108  Real pressure = getPgivenRTh(rhotheta);
109  Real c = std::sqrt(Gamma * pressure / rho);
110 
111  // If we are doing implicit acoustic substepping, then the z-direction does not contribute
112  // to the computation of the time step
113  if (l_substepping) {
114  if ((nxc > 1) && (nyc==1)) {
115  // 2-D in x-z
116  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
117  } else if ((nyc > 1) && (nxc==1)) {
118  // 2-D in y-z
119  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
120  } else {
121  // 3-D
122  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
123  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
124  }
125 
126  // If we are not doing implicit acoustic substepping, then the z-direction contributes
127  // to the computation of the time step
128  } else {
129  if (nxc > 1 && nyc > 1) {
130  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
131  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
132  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
133  } else if (nxc > 1) {
134  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
135  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
136  } else if (nyc > 1) {
137  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
138  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
139  } else {
140  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
141  }
142 
143  }
144  }
145  });
146  return new_comp_dt;
147  });
148 
149  } else {
150  estdt_comp_inv = ReduceMax(S_new, ccvel, 0,
151  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
152  Array4<Real const> const& s,
153  Array4<Real const> const& u) -> Real
154  {
155  Real new_comp_dt = -1.e100;
156  amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
157  {
158  {
159  const Real rho = s(i, j, k, Rho_comp);
160  const Real rhotheta = s(i, j, k, RhoTheta_comp);
161 
162  // NOTE: even when moisture is present,
163  // we only use the partial pressure of the dry air
164  // to compute the soundspeed
165  Real pressure = getPgivenRTh(rhotheta);
166  Real c = std::sqrt(Gamma * pressure / rho);
167 
168  // If we are doing implicit acoustic substepping, then the z-direction does not contribute
169  // to the computation of the time step
170  if (l_substepping) {
171  if ((nxc > 1) && (nyc==1)) {
172  // 2-D in x-z
173  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
174  } else if ((nyc > 1) && (nxc==1)) {
175  // 2-D in y-z
176  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
177  } else {
178  // 3-D
179  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
180  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
181  }
182 
183  // If we are not doing implicit acoustic substepping, then the z-direction contributes
184  // to the computation of the time step
185  } else {
186  if (nxc > 1 && nyc > 1) {
187  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
188  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
189  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
190  } else if (nxc > 1) {
191  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
192  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
193  } else if (nyc > 1) {
194  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
195  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
196  } else {
197  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
198  }
199 
200  }
201  }
202  });
203  return new_comp_dt;
204  });
205  } // not EB
206 
207  ParallelDescriptor::ReduceRealMax(estdt_comp_inv);
208  estdt_comp = cfl / estdt_comp_inv;
209 
210  Real estdt_lowM_inv = ReduceMax(ccvel, 0,
211  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
212  Array4<Real const> const& u) -> Real
213  {
214  Real new_lm_dt = -1.e100;
215  Loop(b, [=,&new_lm_dt] (int i, int j, int k) noexcept
216  {
217  new_lm_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0)))*dxinv[0]),
218  ((amrex::Math::abs(u(i,j,k,1)))*dxinv[1]),
219  ((amrex::Math::abs(u(i,j,k,2)))*dxinv[2]), new_lm_dt);
220  });
221  return new_lm_dt;
222  });
223 
224  ParallelDescriptor::ReduceRealMax(estdt_lowM_inv);
225  if (estdt_lowM_inv > 0.0_rt)
226  estdt_lowM = cfl / estdt_lowM_inv;
227 
228  if (verbose) {
229  if (fixed_dt[level] <= 0.0) {
230  Print() << "Using cfl = " << cfl << " and dx/dy/dz_min = " <<
231  1.0/dxinv[0] << " " << 1.0/dxinv[1] << " " << dz_min[level] << std::endl;
232  Print() << "Compressible dt at level " << level << ": " << estdt_comp << std::endl;
233  if (estdt_lowM_inv > 0.0_rt) {
234  Print() << "Anelastic dt at level " << level << ": " << estdt_lowM << std::endl;
235  } else {
236  Print() << "Anelastic dt at level " << level << ": undefined " << std::endl;
237  }
238  }
239 
240  if (fixed_dt[level] > 0.0) {
241  Print() << "Based on cfl of 1.0 " << std::endl;
242  Print() << "Compressible dt at level " << level << " would be: " << estdt_comp/cfl << std::endl;
243  if (estdt_lowM_inv > 0.0_rt) {
244  Print() << "Anelastic dt at level " << level << " would be: " << estdt_lowM/cfl << std::endl;
245  } else {
246  Print() << "Anelastic dt at level " << level << " would be undefined " << std::endl;
247  }
248  Print() << "Fixed dt at level " << level << " is: " << fixed_dt[level] << std::endl;
249  if (fixed_fast_dt[level] > 0.0) {
250  Print() << "Fixed fast dt at level " << level << " is: " << fixed_fast_dt[level] << std::endl;
251  }
252  }
253  }
254 
255  if (solverChoice.substepping_type[level] != SubsteppingType::None) {
256  if (fixed_dt[level] > 0. && fixed_fast_dt[level] > 0.) {
257  dt_fast_ratio = static_cast<long>( fixed_dt[level] / fixed_fast_dt[level] );
258  } else if (fixed_dt[level] > 0.) {
259  // Max CFL_c = 1.0 for substeps by default, but we enforce a min of 4 substeps
260  auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
261  dt_fast_ratio = static_cast<long>( std::max(fixed_dt[level]/dt_sub_max,4.) );
262  } else {
263  // auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
264  // dt_fast_ratio = static_cast<long>( std::max(estdt_comp/dt_sub_max,4.) );
265  dt_fast_ratio = static_cast<long>( std::max(cfl / sub_cfl, 4.) );
266  }
267 
268  // Force time step ratio to be an even value
270  if ( dt_fast_ratio%2 != 0) dt_fast_ratio += 1;
271  } else {
272  if ( dt_fast_ratio%6 != 0) {
273  Print() << "mri_dt_ratio = " << dt_fast_ratio
274  << " not divisible by 6 for N/3 substeps in stage 1" << std::endl;
275  dt_fast_ratio = static_cast<int>(std::ceil(dt_fast_ratio/6.0) * 6);
276  }
277  }
278 
279  if (verbose) {
280  Print() << "smallest even ratio is: " << dt_fast_ratio << std::endl;
281  }
282  } // if substepping
283 
284  if (fixed_dt[level] > 0.0) {
285  return fixed_dt[level];
286  } else {
287  // Anelastic (substepping is not allowed)
288  if (l_anelastic) {
289 
290  // Make sure that timestep is less than the dt_max
291  estdt_lowM = amrex::min(estdt_lowM, dt_max);
292 
293  // On the first timestep enforce dt_max_initial
294  if(istep[level] == 0){
295  return amrex::min(dt_max_initial, estdt_lowM);
296  }
297  else{
298  return estdt_lowM;
299  }
300 
301 
302  // Compressible with or without substepping
303  } else {
304  return estdt_comp;
305  }
306  }
307 }
constexpr amrex::Real Gamma
Definition: ERF_Constants.H:19
amrex::Vector< amrex::Real > dz_min
Definition: ERF.H:1345
amrex::Vector< amrex::Real > fixed_dt
Definition: ERF.H:1051
static amrex::Real dt_max
Definition: ERF.H:1048
amrex::Vector< amrex::Real > fixed_fast_dt
Definition: ERF.H:1052
static amrex::Real cfl
Definition: ERF.H:1043
static amrex::Real sub_cfl
Definition: ERF.H:1044
Definition: ERF_EB.H:13
int force_stage1_single_substep
Definition: ERF_DataStruct.H:924
amrex::Vector< SubsteppingType > substepping_type
Definition: ERF_DataStruct.H:926
Here is the call graph for this function:

◆ Evolve()

void ERF::Evolve ( )
523 {
524  BL_PROFILE_VAR("ERF::Evolve()", evolve);
525 
526  Real cur_time = t_new[0];
527 
528  // Take one coarse timestep by calling timeStep -- which recursively calls timeStep
529  // for finer levels (with or without subcycling)
530  for (int step = istep[0]; step < max_step && start_time+cur_time < stop_time; ++step)
531  {
532  if (use_datetime) {
533  Print() << "\n" << getTimestamp(start_time+cur_time, datetime_format)
534  << " (" << cur_time << " s elapsed)" << std::endl;
535  }
536  Print() << "\nCoarse STEP " << step+1 << " starts ..." << std::endl;
537 
538  ComputeDt(step);
539 
540  // Make sure we have read enough of the boundary plane data to make it through this timestep
541  if (input_bndry_planes)
542  {
543  m_r2d->read_input_files(cur_time,dt[0],m_bc_extdir_vals);
544  }
545 
546 #ifdef ERF_USE_PARTICLES
547  // We call this every time step with the knowledge that the particles may be
548  // initialized at a later time than the simulation start time.
549  // The ParticleContainer carries a "start time" so the initialization will happen
550  // only when a) time > start_time, and b) particles have not yet been initialized
551  initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,cur_time);
552 #endif
553 
554  auto dEvolveTime0 = amrex::second();
555 
556  int iteration = 1;
557  timeStep(0, cur_time, iteration);
558 
559  cur_time += dt[0];
560 
561  Print() << "Coarse STEP " << step+1 << " ends." << " TIME = " << cur_time
562  << " DT = " << dt[0] << std::endl;
563 
564  if (check_for_nans > 0) {
565  amrex::Print() << "Testing new state and vels for NaNs at end of timestep" << std::endl;
566  for (int lev = 0; lev <= finest_level; ++lev) {
569  }
570  }
571 
572  if (verbose > 0)
573  {
574  auto dEvolveTime = amrex::second() - dEvolveTime0;
575  ParallelDescriptor::ReduceRealMax(dEvolveTime,ParallelDescriptor::IOProcessorNumber());
576  amrex::Print() << "Timestep time = " << dEvolveTime << " seconds." << '\n';
577  }
578 
579  post_timestep(step, cur_time, dt[0]);
580 
581  if (writeNow(cur_time, step+1, m_plot3d_int_1, m_plot3d_per_1, dt[0], last_plot3d_file_time_1)) {
582  last_plot3d_file_step_1 = step+1;
584  for (int lev = 0; lev <= finest_level; ++lev) {lsm.Plot(lev, step+1);}
586  }
587  if (writeNow(cur_time, step+1, m_plot3d_int_2, m_plot3d_per_2, dt[0], last_plot3d_file_time_2)) {
588  last_plot3d_file_step_2 = step+1;
590  for (int lev = 0; lev <= finest_level; ++lev) {lsm.Plot(lev, step+1);}
592  }
593 
594  if (writeNow(cur_time, step+1, m_plot2d_int_1, m_plot2d_per_1, dt[0], last_plot2d_file_time_1)) {
595  last_plot2d_file_step_1 = step+1;
598  }
599 
600  if (writeNow(cur_time, step+1, m_plot2d_int_2, m_plot2d_per_2, dt[0], last_plot2d_file_time_2)) {
601  last_plot2d_file_step_2 = step+1;
604  }
605 
606  if (writeNow(cur_time, step+1, m_subvol_int, m_subvol_per, dt[0], last_subvol_time)) {
607  last_subvol_step = step+1;
610  }
611 
612  if (writeNow(cur_time, step+1, m_check_int, m_check_per, dt[0], last_check_file_time)) {
613  last_check_file_step = step+1;
616  }
617 
618 #ifdef AMREX_MEM_PROFILING
619  {
620  std::ostringstream ss;
621  ss << "[STEP " << step+1 << "]";
622  MemProfiler::report(ss.str());
623  }
624 #endif
625 
626  if (cur_time >= stop_time - 1.e-6*dt[0]) break;
627  }
628 
629  // Write plotfiles at final time
630  if ( (m_plot3d_int_1 > 0 || m_plot3d_per_1 > 0.) && istep[0] > last_plot3d_file_step_1 ) {
633  }
634  if ( (m_plot3d_int_2 > 0 || m_plot3d_per_2 > 0.) && istep[0] > last_plot3d_file_step_2) {
637  }
638  if ( (m_plot2d_int_1 > 0 || m_plot2d_per_1 > 0.) && istep[0] > last_plot2d_file_step_1 ) {
641  }
642  if ( (m_plot2d_int_2 > 0 || m_plot2d_per_2 > 0.) && istep[0] > last_plot2d_file_step_2) {
645  }
646  if ( (m_subvol_int > 0 || m_subvol_per > 0.) && istep[0] > last_subvol_step) {
649  }
650 
651  if ( (m_check_int > 0 || m_check_per > 0.) && istep[0] > last_check_file_step) {
654  }
655 
656  BL_PROFILE_VAR_STOP(evolve);
657 }
AMREX_FORCE_INLINE std::string getTimestamp(const amrex::Real epoch_real, const std::string &datetime_format)
Definition: ERF_EpochTime.H:72
static int last_check_file_step
Definition: ERF.H:1008
static int last_subvol_step
Definition: ERF.H:1009
int max_step
Definition: ERF.H:1030
static amrex::Real last_plot2d_file_time_2
Definition: ERF.H:1014
amrex::Vector< std::string > subvol3d_var_names
Definition: ERF.H:1093
amrex::Real m_plot2d_per_1
Definition: ERF.H:1077
static amrex::Real last_plot2d_file_time_1
Definition: ERF.H:1013
static int last_plot2d_file_step_2
Definition: ERF.H:1007
int m_subvol_int
Definition: ERF.H:1074
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max > m_bc_extdir_vals
Definition: ERF.H:985
static amrex::Real last_plot3d_file_time_2
Definition: ERF.H:1012
int m_plot2d_int_2
Definition: ERF.H:1073
int m_plot3d_int_1
Definition: ERF.H:1070
static int last_plot3d_file_step_2
Definition: ERF.H:1005
void post_timestep(int nstep, amrex::Real time, amrex::Real dt_lev)
Definition: ERF.cpp:661
amrex::Real m_subvol_per
Definition: ERF.H:1079
amrex::Real m_plot2d_per_2
Definition: ERF.H:1078
amrex::Real m_check_per
Definition: ERF.H:1091
int m_check_int
Definition: ERF.H:1090
static int input_bndry_planes
Definition: ERF.H:1267
static amrex::Real last_subvol_time
Definition: ERF.H:1016
void Write2DPlotFile(int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile.cpp:1919
const std::string datetime_format
Definition: ERF.H:1037
bool use_datetime
Definition: ERF.H:1036
void ComputeDt(int step=-1)
Definition: ERF_ComputeTimestep.cpp:11
amrex::Real m_plot3d_per_2
Definition: ERF.H:1076
static PlotFileType plotfile3d_type_2
Definition: ERF.H:1208
static PlotFileType plotfile2d_type_2
Definition: ERF.H:1210
bool writeNow(const amrex::Real cur_time, const int nstep, const int plot_int, const amrex::Real plot_per, const amrex::Real dt_0, amrex::Real &last_file_time)
Definition: ERF.cpp:2766
int m_plot2d_int_1
Definition: ERF.H:1072
void WriteCheckpointFile() const
Definition: ERF_Checkpoint.cpp:26
void Write3DPlotFile(int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile.cpp:307
static int last_plot2d_file_step_1
Definition: ERF.H:1006
amrex::Real m_plot3d_per_1
Definition: ERF.H:1075
std::unique_ptr< ReadBndryPlanes > m_r2d
Definition: ERF.H:1328
static amrex::Real last_check_file_time
Definition: ERF.H:1015
static int last_plot3d_file_step_1
Definition: ERF.H:1004
static amrex::Real last_plot3d_file_time_1
Definition: ERF.H:1011
void WriteSubvolume(amrex::Vector< std::string > subvol_var_names)
Definition: ERF_WriteSubvolume.cpp:144
static PlotFileType plotfile2d_type_1
Definition: ERF.H:1209
static PlotFileType plotfile3d_type_1
Definition: ERF.H:1207
int m_plot3d_int_2
Definition: ERF.H:1071
void timeStep(int lev, amrex::Real time, int iteration)
Definition: ERF_TimeStep.cpp:17
void Plot(const int &lev, const int &nstep)
Definition: ERF_LandSurface.H:71

Referenced by main().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ fill_from_bndryregs()

void ERF::fill_from_bndryregs ( const amrex::Vector< amrex::MultiFab * > &  mfs,
amrex::Real  time 
)
14 {
15  //
16  // We now assume that if we read in on one face, we read in on all faces
17  //
18  AMREX_ALWAYS_ASSERT(m_r2d);
19 
20  int lev = 0;
21  const Box& domain = geom[lev].Domain();
22 
23  const auto& dom_lo = lbound(domain);
24  const auto& dom_hi = ubound(domain);
25 
26  Vector<std::unique_ptr<PlaneVector>>& bndry_data = m_r2d->interp_in_time(time);
27 
28  const BCRec* bc_ptr = domain_bcs_type_d.data();
29 
30  // xlo: ori = 0
31  // ylo: ori = 1
32  // zlo: ori = 2
33  // xhi: ori = 3
34  // yhi: ori = 4
35  // zhi: ori = 5
36  const auto& bdatxlo = (*bndry_data[0])[lev].const_array();
37  const auto& bdatylo = (*bndry_data[1])[lev].const_array();
38  const auto& bdatxhi = (*bndry_data[3])[lev].const_array();
39  const auto& bdatyhi = (*bndry_data[4])[lev].const_array();
40 
41  int bccomp;
42 
43  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx)
44  {
45  MultiFab& mf = *mfs[var_idx];
46  const int icomp = 0;
47  const int ncomp = mf.nComp();
48 
49  if (var_idx == Vars::xvel) {
50  bccomp = BCVars::xvel_bc;
51  } else if (var_idx == Vars::yvel) {
52  bccomp = BCVars::yvel_bc;
53  } else if (var_idx == Vars::zvel) {
54  bccomp = BCVars::zvel_bc;
55  } else if (var_idx == Vars::cons) {
56  bccomp = BCVars::cons_bc;
57  }
58 
59 #ifdef AMREX_USE_OMP
60 #pragma omp parallel if (Gpu::notInLaunchRegion())
61 #endif
62  for (MFIter mfi(mf); mfi.isValid(); ++mfi)
63  {
64  const Array4<Real>& dest_arr = mf.array(mfi);
65  Box bx = mfi.growntilebox();
66 
67  // x-faces
68  {
69  Box bx_xlo(bx); bx_xlo.setBig(0,dom_lo.x-1);
70  if (var_idx == Vars::xvel) bx_xlo.setBig(0,dom_lo.x);
71 
72  Box bx_xhi(bx); bx_xhi.setSmall(0,dom_hi.x+1);
73  if (var_idx == Vars::xvel) bx_xhi.setSmall(0,dom_hi.x);
74 
75  ParallelFor(
76  bx_xlo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
77  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
78  BCVars::RhoScalar_bc_comp : icomp+n;
79  if (bc_ptr[bc_comp].lo(0) == ERFBCType::ext_dir_ingested) {
80  int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
81  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
82  dest_arr(i,j,k,icomp+n) = bdatxlo(dom_lo.x-1,jb,kb,bccomp+n);
83  }
84  },
85  bx_xhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
86  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
87  BCVars::RhoScalar_bc_comp : icomp+n;
88  if (bc_ptr[bc_comp].hi(0) == ERFBCType::ext_dir_ingested) {
89  int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
90  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
91  dest_arr(i,j,k,icomp+n) = bdatxhi(dom_hi.x+1,jb,kb,bccomp+n);
92  }
93  }
94  );
95  } // x-faces
96 
97  // y-faces
98  {
99  Box bx_ylo(bx); bx_ylo.setBig (1,dom_lo.y-1);
100  if (var_idx == Vars::yvel) bx_ylo.setBig(1,dom_lo.y);
101 
102  Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+1);
103  if (var_idx == Vars::yvel) bx_yhi.setSmall(1,dom_hi.y);
104 
105  ParallelFor(
106  bx_ylo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
107  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
108  BCVars::RhoScalar_bc_comp : icomp+n;
109  if (bc_ptr[bc_comp].lo(1) == ERFBCType::ext_dir_ingested) {
110  int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
111  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
112  dest_arr(i,j,k,icomp+n) = bdatylo(ib,dom_lo.y-1,kb,bccomp+n);
113  }
114  },
115  bx_yhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
116  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
117  BCVars::RhoScalar_bc_comp : icomp+n;
118  if (bc_ptr[bc_comp].hi(1) == ERFBCType::ext_dir_ingested) {
119  int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
120  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
121  dest_arr(i,j,k,icomp+n) = bdatyhi(ib,dom_hi.y+1,kb,bccomp+n);
122  }
123  }
124  );
125  } // y-faces
126  } // mf
127  } // var_idx
128 }
#define RhoScalar_comp
Definition: ERF_IndexDefines.H:40
#define NSCALARS
Definition: ERF_IndexDefines.H:16
amrex::Gpu::DeviceVector< amrex::BCRec > domain_bcs_type_d
Definition: ERF.H:979
@ RhoScalar_bc_comp
Definition: ERF_IndexDefines.H:80
@ ext_dir_ingested
Definition: ERF_IndexDefines.H:212

◆ fill_rhs()

void ERF::fill_rhs ( amrex::MultiFab &  rhs_mf,
const amrex::MultiFab &  state_mf,
amrex::Real  time,
const amrex::Geometry &  geom 
)
private

◆ FillBdyCCVels()

void ERF::FillBdyCCVels ( amrex::Vector< amrex::MultiFab > &  mf_cc_vel,
int  levc = 0 
)
12 {
13  // Impose bc's at domain boundaries
14  for (int ilev(0); ilev < mf_cc_vel.size(); ++ilev)
15  {
16  int lev = ilev + levc;
17  Box domain(Geom(lev).Domain());
18 
19  int ihi = domain.bigEnd(0);
20  int jhi = domain.bigEnd(1);
21  int khi = domain.bigEnd(2);
22 
23  // Impose periodicity first
24  mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
25 
26  int jper = (Geom(lev).isPeriodic(1));
27  int kper = (Geom(lev).isPeriodic(2));
28 
29  for (MFIter mfi(mf_cc_vel[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
30  {
31  const Box& bx = mfi.tilebox();
32  const Array4<Real>& vel_arr = mf_cc_vel[lev].array(mfi);
33 
34  if (!Geom(lev).isPeriodic(0)) {
35  // Low-x side
36  if (bx.smallEnd(0) <= domain.smallEnd(0)) {
37  Real multn = ( (phys_bc_type[0] == ERF_BC::slip_wall ) ||
39  (phys_bc_type[0] == ERF_BC::symmetry ) ) ? -1. : 1.;
40  Real multt = (phys_bc_type[0] == ERF_BC::no_slip_wall) ? -1. : 1.;
41  Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
42  ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
43  {
44  vel_arr(-1,j,k,0) = multn*vel_arr(0,j,k,0); // u
45  vel_arr(-1,j,k,1) = multt*vel_arr(0,j,k,1); // v
46  vel_arr(-1,j,k,2) = multt*vel_arr(0,j,k,2); // w
47  });
48  }
49 
50  // High-x side
51  if (bx.bigEnd(0) >= domain.bigEnd(0)) {
52  Real multn = ( (phys_bc_type[3] == ERF_BC::slip_wall ) ||
54  (phys_bc_type[3] == ERF_BC::symmetry ) ) ? -1. : 1.;
55  Real multt = (phys_bc_type[3] == ERF_BC::no_slip_wall) ? -1. : 1.;
56  Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
57  ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
58  {
59  vel_arr(ihi+1,j,k,0) = multn*vel_arr(ihi,j,k,0); // u
60  vel_arr(ihi+1,j,k,1) = multt*vel_arr(ihi,j,k,1); // v
61  vel_arr(ihi+1,j,k,2) = multt*vel_arr(ihi,j,k,2); // w
62  });
63  }
64  } // !periodic
65 
66  if (!Geom(lev).isPeriodic(1)) {
67  // Low-y side
68  if (bx.smallEnd(1) <= domain.smallEnd(1)) {
69  Real multn = ( (phys_bc_type[1] == ERF_BC::slip_wall ) ||
71  (phys_bc_type[1] == ERF_BC::symmetry ) ) ? -1. : 1.;
72  Real multt = (phys_bc_type[1] == ERF_BC::no_slip_wall) ? -1. : 1.;
73  Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
74  ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
75  {
76  vel_arr(i,-1,k,0) = multt*vel_arr(i,0,k,0); // u
77  vel_arr(i,-1,k,1) = multn*vel_arr(i,0,k,1); // u
78  vel_arr(i,-1,k,2) = multt*vel_arr(i,0,k,2); // w
79  });
80  }
81 
82  // High-y side
83  if (bx.bigEnd(1) >= domain.bigEnd(1)) {
84  Real multn = ( (phys_bc_type[4] == ERF_BC::slip_wall ) ||
86  (phys_bc_type[4] == ERF_BC::symmetry ) ) ? -1. : 1.;
87  Real multt = (phys_bc_type[4] == ERF_BC::no_slip_wall) ? -1. : 1.;
88  Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
89  ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
90  {
91  vel_arr(i,jhi+1,k,0) = multt*vel_arr(i,jhi,k,0); // u
92  vel_arr(i,jhi+1,k,1) = multn*vel_arr(i,jhi,k,1); // v
93  vel_arr(i,jhi+1,k,2) = multt*vel_arr(i,jhi,k,2); // w
94  });
95  }
96  } // !periodic
97 
98  if (!Geom(lev).isPeriodic(2)) {
99  // Low-z side
100  if (bx.smallEnd(2) <= domain.smallEnd(2)) {
101  Real multn = ( (phys_bc_type[2] == ERF_BC::slip_wall ) ||
103  (phys_bc_type[2] == ERF_BC::symmetry ) ) ? -1. : 1.;
104  Real multt = (phys_bc_type[2] == ERF_BC::no_slip_wall) ? -1. : 1.;
105  Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
106  ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
107  {
108  vel_arr(i,j,-1,0) = multt*vel_arr(i,j,0,0); // u
109  vel_arr(i,j,-1,1) = multt*vel_arr(i,j,0,1); // v
110  vel_arr(i,j,-1,2) = multn*vel_arr(i,j,0,2); // w
111  });
112  }
113 
114  // High-z side
115  if (bx.bigEnd(2) >= domain.bigEnd(2)) {
116  Real multn = ( (phys_bc_type[5] == ERF_BC::slip_wall ) ||
118  (phys_bc_type[5] == ERF_BC::symmetry ) ) ? -1. : 1.;
119  Real multt = (phys_bc_type[5] == ERF_BC::no_slip_wall) ? -1. : 1.;
120  Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
121  ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
122  {
123  vel_arr(i,j,khi+1,0) = multt*vel_arr(i,j,khi,0); // u
124  vel_arr(i,j,khi+1,1) = multt*vel_arr(i,j,khi,1); // v
125  vel_arr(i,j,khi+1,2) = multn*vel_arr(i,j,khi,2); // w
126  });
127  }
128  } // !periodic
129  } // MFIter
130 
131  // Impose periodicity again
132  mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
133  } // lev
134 }
@ no_slip_wall

◆ FillCoarsePatch()

void ERF::FillCoarsePatch ( int  lev,
amrex::Real  time 
)
private
22 {
23  BL_PROFILE_VAR("FillCoarsePatch()",FillCoarsePatch);
24  AMREX_ASSERT(lev > 0);
25 
26  //
27  //****************************************************************************************************************
28  // First fill velocities and density at the COARSE level so we can convert velocity to momenta at the COARSE level
29  //****************************************************************************************************************
30  //
31  bool cons_only = false;
32  if (lev == 1) {
33  FillPatchCrseLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
34  &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
35  cons_only);
36  } else {
37  FillPatchFineLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
38  &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
39  {&vars_new[lev-1][Vars::cons],
40  &rU_new[lev-1], &rV_new[lev-1], &rW_new[lev-1]},
41  base_state[lev-1], base_state[lev-1],
42  false, cons_only);
43  }
44 
45  //
46  // ************************************************
47  // Convert velocity to momentum at the COARSE level
48  // ************************************************
49  //
50  VelocityToMomentum(vars_new[lev-1][Vars::xvel], IntVect{0},
51  vars_new[lev-1][Vars::yvel], IntVect{0},
52  vars_new[lev-1][Vars::zvel], IntVect{0},
53  vars_new[lev-1][Vars::cons],
54  rU_new[lev-1],
55  rV_new[lev-1],
56  rW_new[lev-1],
57  Geom(lev).Domain(),
59  //
60  // *****************************************************************
61  // Interpolate all cell-centered variables from coarse to fine level
62  // *****************************************************************
63  //
64  Interpolater* mapper_c = &cell_cons_interp;
65  Interpolater* mapper_f = &face_cons_linear_interp;
66 
67  //
68  //************************************************************************************************
69  // Interpolate cell-centered data from coarse to fine level
70  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
71  // ************************************************************************************************
72  IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
73  int ncomp_cons = vars_new[lev][Vars::cons].nComp();
74 
75  InterpFromCoarseLevel(vars_new[lev ][Vars::cons], ngvect_cons, IntVect(0,0,0),
76  vars_new[lev-1][Vars::cons], 0, 0, ncomp_cons,
77  geom[lev-1], geom[lev],
78  refRatio(lev-1), mapper_c, domain_bcs_type, BCVars::cons_bc);
79 
80  // ***************************************************************************
81  // Physical bc's for cell centered variables at domain boundary
82  // ***************************************************************************
84  0,ncomp_cons,ngvect_cons,time,BCVars::cons_bc,true);
85 
86  //
87  //************************************************************************************************
88  // Interpolate x-momentum from coarse to fine level
89  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
90  // ************************************************************************************************
91  //
92  InterpFromCoarseLevel(rU_new[lev], IntVect{0}, IntVect{0}, rU_new[lev-1], 0, 0, 1,
93  geom[lev-1], geom[lev],
94  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::xvel_bc);
95 
96  //
97  //************************************************************************************************
98  // Interpolate y-momentum from coarse to fine level
99  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
100  // ************************************************************************************************
101  //
102  InterpFromCoarseLevel(rV_new[lev], IntVect{0}, IntVect{0}, rV_new[lev-1], 0, 0, 1,
103  geom[lev-1], geom[lev],
104  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::yvel_bc);
105 
106  //************************************************************************************************
107  // Interpolate z-momentum from coarse to fine level
108  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
109  // ************************************************************************************************
110  InterpFromCoarseLevel(rW_new[lev], IntVect{0}, IntVect{0}, rW_new[lev-1], 0, 0, 1,
111  geom[lev-1], geom[lev],
112  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::zvel_bc);
113  //
114  // *********************************************************
115  // After interpolation of momentum, convert back to velocity
116  // *********************************************************
117  //
118  for (int which_lev = lev-1; which_lev <= lev; which_lev++)
119  {
121  vars_new[which_lev][Vars::yvel],
122  vars_new[which_lev][Vars::zvel],
123  vars_new[which_lev][Vars::cons],
124  rU_new[which_lev],
125  rV_new[which_lev],
126  rW_new[which_lev],
127  Geom(lev).Domain(),
129  }
130 
131  // ***************************************************************************
132  // Physical bc's at domain boundary
133  // ***************************************************************************
134  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
135 
137  ngvect_vels,time,BCVars::xvel_bc,true);
139  ngvect_vels,time,BCVars::yvel_bc,true);
141  ngvect_vels,time,BCVars::zvel_bc,true);
142 
143  // ***************************************************************************
144  // Since lev > 0 here we don't worry about m_r2d or wrfbdy data
145  // ***************************************************************************
146 }
void FillCoarsePatch(int lev, amrex::Real time)
Definition: ERF_FillCoarsePatch.cpp:21
Here is the call graph for this function:

◆ FillIntermediatePatch()

void ERF::FillIntermediatePatch ( int  lev,
amrex::Real  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
const amrex::Vector< amrex::MultiFab * > &  mfs_mom,
int  ng_cons,
int  ng_vel,
bool  cons_only,
int  icomp_cons,
int  ncomp_cons 
)
private
33 {
34  BL_PROFILE_VAR("FillIntermediatePatch()",FillIntermediatePatch);
35  Interpolater* mapper;
36 
37  PhysBCFunctNoOp null_bc;
38 
39  //
40  // ***************************************************************************
41  // The first thing we do is interpolate the momenta on the "valid" faces of
42  // the fine grids (where the interface is coarse/fine not fine/fine) -- this
43  // will not be over-written by interpolation below because the FillPatch
44  // operators see these as valid faces. But we must have these interpolated
45  // values in the fine data before we call FillPatchTwoLevels.
46  //
47  // Also -- note that we might be filling values by interpolation at physical boundaries
48  // here but that's ok because we will overwrite those values when we impose
49  // the physical bc's below
50  // ***************************************************************************
51  if (lev>0) {
52  if (cf_set_width > 0) {
53  // We note that mfs_vel[Vars::cons] and mfs_mom[Vars::cons] are in fact the same pointer
54  FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
55  }
56  if ( !cons_only && (cf_set_width >= 0) ) {
57  FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
58  FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
59  FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
60  }
61  }
62 
63  // amrex::Print() << "LEVEL " << lev << " CONS ONLY " << cons_only <<
64  // " ICOMP NCOMP " << icomp_cons << " " << ncomp_cons << " NGHOST " << ng_cons << std::endl;
65 
66  if (!cons_only) {
67  AMREX_ALWAYS_ASSERT(mfs_mom.size() == IntVars::NumTypes);
68  AMREX_ALWAYS_ASSERT(mfs_vel.size() == Vars::NumTypes);
69  }
70 
71  // Enforce no penetration for thin immersed body
72  if (!cons_only) {
73  // Enforce no penetration for thin immersed body
74  if (xflux_imask[lev]) {
75  ApplyMask(*mfs_mom[IntVars::xmom], *xflux_imask[lev]);
76  }
77  if (yflux_imask[lev]) {
78  ApplyMask(*mfs_mom[IntVars::ymom], *yflux_imask[lev]);
79  }
80  if (zflux_imask[lev]) {
81  ApplyMask(*mfs_mom[IntVars::zmom], *zflux_imask[lev]);
82  }
83  }
84 
85  //
86  // We now start working on conserved quantities + VELOCITY
87  //
88  if (lev == 0)
89  {
90  // We don't do anything here because we will call the physbcs routines below,
91  // which calls FillBoundary and fills other domain boundary conditions
92  // Physical boundaries will be filled below
93 
94  if (!cons_only)
95  {
96  // ***************************************************************************
97  // We always come in to this call with updated momenta but we need to create updated velocity
98  // in order to impose the rest of the bc's
99  // ***************************************************************************
100  // This only fills VALID region of velocity
101  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
102  *mfs_vel[Vars::cons],
103  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
104  Geom(lev).Domain(), domain_bcs_type);
105  }
106  }
107  else
108  {
109  //
110  // We must fill a temporary then copy it back so we don't double add/subtract
111  //
112  MultiFab mf(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
113  mfs_vel[Vars::cons]->nComp() ,mfs_vel[Vars::cons]->nGrowVect());
114  //
115  // Set all components to 1.789e19, then copy just the density from *mfs_vel[Vars::cons]
116  //
117  mf.setVal(1.789e19);
118  MultiFab::Copy(mf,*mfs_vel[Vars::cons],Rho_comp,Rho_comp,1,mf.nGrowVect());
119 
120  Vector<MultiFab*> fmf = {mfs_vel[Vars::cons],mfs_vel[Vars::cons]};
121  Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
122  Vector<Real> ctime = {t_old[lev-1], t_new[lev-1]};
123  Vector<Real> ftime = {time,time};
124 
125  if (interpolation_type == StateInterpType::Perturbational)
126  {
127  if (icomp_cons+ncomp_cons > 1)
128  {
129  // Divide (rho theta) by rho to get theta
130  MultiFab::Divide(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
131 
132  // Subtract theta_0 from theta
133  MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
134 
135  if (!amrex::almostEqual(time,ctime[1])) {
136  MultiFab::Divide(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
137  Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
138  MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
139  BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
140  }
141  if (!amrex::almostEqual(time,ctime[0])) {
142  MultiFab::Divide(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
143  Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
144  MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
145  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
146  }
147  }
148 
149  // Subtract rho_0 from rho before we interpolate -- note we only subtract
150  // on valid region of mf since the ghost cells will be filled below
151  if (icomp_cons == 0)
152  {
153  MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{0});
154 
155  if (!amrex::almostEqual(time,ctime[1])) {
156  MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
157  BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
158  }
159  if (!amrex::almostEqual(time,ctime[0])) {
160  MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
161  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
162  }
163  }
164  } // interpolation_type == StateInterpType::Perturbational
165 
166  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
167  mapper = &cell_cons_interp;
168  FillPatchTwoLevels(mf, IntVect{ng_cons}, IntVect(0,0,0),
169  time, cmf, ctime, fmf, ftime,
170  icomp_cons, icomp_cons, ncomp_cons, geom[lev-1], geom[lev],
171  refRatio(lev-1), mapper, domain_bcs_type,
172  icomp_cons);
173 
174  if (interpolation_type == StateInterpType::Perturbational)
175  {
176  if (icomp_cons == 0)
177  {
178  // Restore the coarse values to what they were
179  if (!amrex::almostEqual(time,ctime[1])) {
180  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
181  BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
182  }
183  if (!amrex::almostEqual(time,ctime[0])) {
184  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
185  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
186  }
187 
188  // Set values in the cells outside the domain boundary so that we can do the Add
189  // without worrying about uninitialized values outside the domain -- these
190  // will be filled in the physbcs call
191  mf.setDomainBndry(1.234e20,Rho_comp,1,geom[lev]);
192 
193  // Add rho_0 back to rho after we interpolate -- on all the valid + ghost region
194  MultiFab::Add(mf, base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{ng_cons});
195  }
196 
197  if (icomp_cons+ncomp_cons > 1)
198  {
199  // Add theta_0 to theta
200  if (!amrex::almostEqual(time,ctime[1])) {
201  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
202  BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
203  MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
204  Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
205  }
206  if (!amrex::almostEqual(time,ctime[0])) {
207  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
208  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
209  MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
210  Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
211  }
212 
213  // Multiply theta by rho to get (rho theta)
214  MultiFab::Multiply(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
215 
216  // Add theta_0 to theta
217  MultiFab::Add(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
218 
219  // Add theta_0 back to theta
220  MultiFab::Add(mf,base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{ng_cons});
221 
222  // Multiply (theta) by rho to get (rho theta)
223  MultiFab::Multiply(mf,mf,Rho_comp,RhoTheta_comp,1,IntVect{ng_cons});
224  }
225  } // interpolation_type == StateInterpType::Perturbational
226 
227  // Impose physical bc's on fine data (note time and 0 are not used)
228  // Note that we do this after the FillPatch because imposing physical bc's on fine ghost
229  // cells that need to be filled from coarse requires that we have done the interpolation first
230  bool do_fb = true; bool do_terrain_adjustment = false;
231  (*physbcs_cons[lev])(mf,*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
232  icomp_cons,ncomp_cons,IntVect{ng_cons},time,BCVars::cons_bc,
233  do_fb, do_terrain_adjustment);
234 
235  // Make sure to only copy back the components we worked on
236  MultiFab::Copy(*mfs_vel[Vars::cons],mf,icomp_cons,icomp_cons,ncomp_cons,IntVect{ng_cons});
237 
238  // *****************************************************************************************
239 
240  if (!cons_only)
241  {
242  // ***************************************************************************
243  // We always come in to this call with updated momenta but we need to create updated velocity
244  // in order to impose the rest of the bc's
245  // ***************************************************************************
246  // This only fills VALID region of velocity
247  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
248  *mfs_vel[Vars::cons],
249  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
250  Geom(lev).Domain(), domain_bcs_type);
251 
252  mapper = &face_cons_linear_interp;
253 
254  //
255  // NOTE: All interpolation here happens on velocities not momenta;
256  // note we only do the interpolation and FillBoundary here,
257  // physical bc's are imposed later
258  //
259  // NOTE: This will only fill velocity from coarse grid *outside* the fine grids
260  // unlike the FillSet calls above which filled momenta on the coarse/fine bdy
261  //
262 
263  MultiFab& mfu = *mfs_vel[Vars::xvel];
264 
265  fmf = {&mfu,&mfu};
266  cmf = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
267 
268  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
269  FillPatchTwoLevels(mfu, IntVect{ng_vel}, IntVect(0,0,0),
270  time, cmf, ctime, fmf, ftime,
271  0, 0, 1, geom[lev-1], geom[lev],
272  refRatio(lev-1), mapper, domain_bcs_type,
274 
275  // *****************************************************************************************
276 
277  MultiFab& mfv = *mfs_vel[Vars::yvel];
278 
279  fmf = {&mfv,&mfv};
280  cmf = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
281 
282  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
283  FillPatchTwoLevels(mfv, IntVect{ng_vel}, IntVect(0,0,0),
284  time, cmf, ctime, fmf, ftime,
285  0, 0, 1, geom[lev-1], geom[lev],
286  refRatio(lev-1), mapper, domain_bcs_type,
288 
289  // *****************************************************************************************
290 
291  MultiFab& mfw = *mfs_vel[Vars::zvel];
292 
293  fmf = {&mfw,&mfw};
294  cmf = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
295 
296  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
297  FillPatchTwoLevels(mfw, IntVect{ng_vel}, IntVect(0,0,0),
298  time, cmf, ctime, fmf, ftime,
299  0, 0, 1, geom[lev-1], geom[lev],
300  refRatio(lev-1), mapper, domain_bcs_type,
302  } // !cons_only
303  } // lev > 0
304 
305  // ***************************************************************************
306  // Physical bc's at domain boundary
307  // ***************************************************************************
308  IntVect ngvect_cons = IntVect(ng_cons,ng_cons,ng_cons);
309  IntVect ngvect_vels = IntVect(ng_vel ,ng_vel ,ng_vel);
310 
311  bool do_fb = true;
312 
313 #ifdef ERF_USE_NETCDF
314  // We call this here because it is an ERF routine
315  if (solverChoice.use_real_bcs && (lev==0)) {
317  fill_from_realbdy_upwind(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
318  } else {
319  fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
320  }
321  do_fb = false;
322  }
323 #endif
324 
325  if (m_r2d) fill_from_bndryregs(mfs_vel,time);
326 
327  // We call this even if use_real_bcs is true because these will fill the vertical bcs
328  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
329  icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
330  if (!cons_only) {
331  (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
332  ngvect_vels,time,BCVars::xvel_bc, do_fb);
333  (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
334  ngvect_vels,time,BCVars::yvel_bc, do_fb);
335  (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
336  ngvect_vels,time,BCVars::zvel_bc, do_fb);
337  }
338  // ***************************************************************************
339 
340  // We always come in to this call with momenta so we need to leave with momenta!
341  // We need to make sure we convert back on all ghost cells/faces because this is
342  // how velocity from fine-fine copies (as well as physical and interpolated bcs) will be filled
343  if (!cons_only)
344  {
345  IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::xvel]->nGrowVect();
346  IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::yvel]->nGrowVect();
347  IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : mfs_vel[Vars::zvel]->nGrowVect();
348 
349  VelocityToMomentum(*mfs_vel[Vars::xvel], ngu,
350  *mfs_vel[Vars::yvel], ngv,
351  *mfs_vel[Vars::zvel], ngw,
352  *mfs_vel[Vars::cons],
353  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
354  Geom(lev).Domain(),
356  }
357 
358  // NOTE: There are not FillBoundary calls here for the following reasons:
359  // Removal of the FillBoundary (FB) calls has bee completed for the following reasons:
360  //
361  // 1. physbc_cons is called before VelocityToMomentum and a FB is completed in that functor.
362  // Therefore, the conserved CC vars have their inter-rank ghost cells filled and then their
363  // domain ghost cells filled from the BC operations. We should not call FB on this MF again.
364  //
365  // 2. physbc_u/v/w is also called before VelocityToMomentum and a FB is completed those functors.
366  // Furthermore, VelocityToMomentum operates on a growntilebox so we exit that routine with momentum
367  // filled everywhere---i.e., physbc_u/v/w fills velocity ghost cells (inter-rank and domain)
368  // and then V2M does the conversion to momenta everywhere; so there is again no need to do a FB on momenta.
369 }
AMREX_GPU_HOST AMREX_FORCE_INLINE void ApplyMask(amrex::MultiFab &dst, const amrex::iMultiFab &imask, const int nghost=0)
Definition: ERF_Utils.H:408
void fill_from_bndryregs(const amrex::Vector< amrex::MultiFab * > &mfs, amrex::Real time)
Definition: ERF_BoundaryConditionsBndryReg.cpp:13
void FillIntermediatePatch(int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons)
Definition: ERF_FillIntermediatePatch.cpp:28
@ NumTypes
Definition: ERF_IndexDefines.H:162
static bool upwind_real_bcs
Definition: ERF_DataStruct.H:907
static bool use_real_bcs
Definition: ERF_DataStruct.H:904
Here is the call graph for this function:

◆ FillPatchCrseLevel()

void ERF::FillPatchCrseLevel ( int  lev,
amrex::Real  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
bool  cons_only = false 
)
private
285 {
286  BL_PROFILE_VAR("ERF::FillPatchCrseLevel()",ERF_FillPatchCrseLevel);
287 
288  AMREX_ALWAYS_ASSERT(lev == 0);
289 
290  IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
291  IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
292 
293  Vector<Real> ftime = {t_old[lev], t_new[lev]};
294 
295  //
296  // Below we call FillPatchSingleLevel which does NOT fill ghost cells outside the domain
297  //
298 
299  Vector<MultiFab*> fmf;
300  Vector<MultiFab*> fmf_u;
301  Vector<MultiFab*> fmf_v;
302  Vector<MultiFab*> fmf_w;
303 
304  if (amrex::almostEqual(time,ftime[0])) {
305  fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
306  } else if (amrex::almostEqual(time,ftime[1])) {
307  fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
308  } else {
309  fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
310  }
311 
312  const int ncomp = mfs_vel[Vars::cons]->nComp();
313 
314  FillPatchSingleLevel(*mfs_vel[Vars::cons], ngvect_cons, time, fmf, IntVect(0,0,0), ftime,
315  0, 0, ncomp, geom[lev]);
316 
317  if (!cons_only) {
318  if (amrex::almostEqual(time,ftime[0])) {
319  fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
320  fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
321  fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
322  } else if (amrex::almostEqual(time,ftime[1])) {
323  fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
324  fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
325  fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
326  } else {
327  fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
328  fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
329  fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
330  }
331  FillPatchSingleLevel(*mfs_vel[Vars::xvel], ngvect_vels, time, fmf_u,
332  IntVect(0,0,0), ftime, 0, 0, 1, geom[lev]);
333 
334  FillPatchSingleLevel(*mfs_vel[Vars::yvel], ngvect_vels, time, fmf_v,
335  IntVect(0,0,0), ftime, 0, 0, 1, geom[lev]);
336 
337  FillPatchSingleLevel(*mfs_vel[Vars::zvel], ngvect_vels, time, fmf_w,
338  IntVect(0,0,0), ftime, 0, 0, 1, geom[lev]);
339  } // !cons_only
340 
341  // ***************************************************************************
342  // Physical bc's at domain boundary
343  // ***************************************************************************
344  int icomp_cons = 0;
345  int ncomp_cons = mfs_vel[Vars::cons]->nComp();
346 
347  bool do_fb = true;
348 
349 #ifdef ERF_USE_NETCDF
350  // We call this here because it is an ERF routine
351  if(solverChoice.use_real_bcs && (lev==0)) {
353  fill_from_realbdy_upwind(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
354  } else {
355  fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
356  }
357  do_fb = false;
358  }
359 #endif
360 
361  if (m_r2d) fill_from_bndryregs(mfs_vel,time);
362 
363  // We call this even if use_real_bcs is true because these will fill the vertical bcs
364  // Note that we call FillBoundary inside the physbcs call
365  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
366  icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
367  if (!cons_only) {
368  (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
369  ngvect_vels,time,BCVars::xvel_bc, do_fb);
370  (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
371  ngvect_vels,time,BCVars::yvel_bc, do_fb);
372  (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
373  ngvect_vels,time,BCVars::zvel_bc, do_fb);
374  }
375 }

◆ FillPatchFineLevel()

void ERF::FillPatchFineLevel ( int  lev,
amrex::Real  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
const amrex::Vector< amrex::MultiFab * > &  mfs_mom,
const amrex::MultiFab &  old_base_state,
const amrex::MultiFab &  new_base_state,
bool  fillset = true,
bool  cons_only = false 
)
private
26 {
27  BL_PROFILE_VAR("ERF::FillPatchFineLevel()",ERF_FillPatchFineLevel);
28 
29  AMREX_ALWAYS_ASSERT(lev > 0);
30 
31  Interpolater* mapper = nullptr;
32 
33  PhysBCFunctNoOp null_bc;
34 
35  //
36  // ***************************************************************************
37  // The first thing we do is interpolate the momenta on the "valid" faces of
38  // the fine grids (where the interface is coarse/fine not fine/fine) -- this
39  // will not be over-written below because the FillPatch operators see these as
40  // valid faces.
41  //
42  // Note that we interpolate momentum not velocity, but all the other boundary
43  // conditions are imposed on velocity, so we convert to momentum here then
44  // convert back.
45  // ***************************************************************************
46  if (fillset) {
47  if (cf_set_width > 0) {
48  FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
49  }
50  if (cf_set_width >= 0 && !cons_only) {
51  VelocityToMomentum(*mfs_vel[Vars::xvel], IntVect{0},
52  *mfs_vel[Vars::yvel], IntVect{0},
53  *mfs_vel[Vars::zvel], IntVect{0},
54  *mfs_vel[Vars::cons],
55  *mfs_mom[IntVars::xmom],
56  *mfs_mom[IntVars::ymom],
57  *mfs_mom[IntVars::zmom],
58  Geom(lev).Domain(),
60 
61  FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
62  FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
63  FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
64 
65  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
66  *mfs_vel[Vars::cons],
67  *mfs_mom[IntVars::xmom],
68  *mfs_mom[IntVars::ymom],
69  *mfs_mom[IntVars::zmom],
70  Geom(lev).Domain(),
72  }
73  }
74 
75  IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
76  IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
77 
78  Vector<Real> ftime = {t_old[lev ], t_new[lev ]};
79  Vector<Real> ctime = {t_old[lev-1], t_new[lev-1]};
80 
81  amrex::Real small_dt = 1.e-8 * (ftime[1] - ftime[0]);
82 
83  Vector<MultiFab*> fmf;
84  if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
85  fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
86  } else if (amrex::almostEqual(time,ftime[1])) {
87  fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
88  } else {
89  fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
90  }
91  Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
92 
93  // We must fill a temporary then copy it back so we don't double add/subtract
94  MultiFab mf_c(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
95  mfs_vel[Vars::cons]->nComp() ,mfs_vel[Vars::cons]->nGrowVect());
96 
97  mapper = &cell_cons_interp;
98 
99  if (interpolation_type == StateInterpType::Perturbational)
100  {
101  // Divide (rho theta) by rho to get theta (before we subtract rho0 from rho!)
102  if (!amrex::almostEqual(time,ctime[1])) {
103  MultiFab::Divide(vars_old[lev-1][Vars::cons],vars_old[lev-1][Vars::cons],
104  Rho_comp,RhoTheta_comp,1,ngvect_cons);
105  MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
106  BaseState::r0_comp,Rho_comp,1,ngvect_cons);
107  MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
108  BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
109  }
110  if (!amrex::almostEqual(time,ctime[0])) {
111  MultiFab::Divide(vars_new[lev-1][Vars::cons],vars_new[lev-1][Vars::cons],
112  Rho_comp,RhoTheta_comp,1,ngvect_cons);
113  MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
114  BaseState::r0_comp,Rho_comp,1,ngvect_cons);
115  MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
116  BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
117  }
118 
119  if (!amrex::almostEqual(time,ftime[1])) {
120  MultiFab::Divide(vars_old[lev ][Vars::cons],vars_old[lev ][Vars::cons],
121  Rho_comp,RhoTheta_comp,1,IntVect{0});
122  MultiFab::Subtract(vars_old[lev ][Vars::cons],old_base_state,
123  BaseState::r0_comp,Rho_comp,1,IntVect{0});
124  MultiFab::Subtract(vars_old[lev ][Vars::cons],old_base_state,
125  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
126  }
127  if (!amrex::almostEqual(time,ftime[0])) {
128  MultiFab::Divide(vars_new[lev ][Vars::cons],vars_new[lev ][Vars::cons],
129  Rho_comp,RhoTheta_comp,1,IntVect{0});
130  MultiFab::Subtract(vars_new[lev ][Vars::cons],old_base_state,
131  BaseState::r0_comp,Rho_comp,1,IntVect{0});
132  MultiFab::Subtract(vars_new[lev ][Vars::cons],old_base_state,
133  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
134  }
135  }
136 
137  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
138  FillPatchTwoLevels(mf_c, ngvect_cons, IntVect(0,0,0),
139  time, cmf, ctime, fmf, ftime,
140  0, 0, mf_c.nComp(), geom[lev-1], geom[lev],
141  refRatio(lev-1), mapper, domain_bcs_type,
143 
144  if (interpolation_type == StateInterpType::Perturbational)
145  {
146  // Restore the coarse values to what they were
147  if (!amrex::almostEqual(time,ctime[1])) {
148  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
149  BaseState::r0_comp,Rho_comp,1,ngvect_cons);
150  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
151  BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
152  MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
153  Rho_comp,RhoTheta_comp,1,ngvect_cons);
154  }
155  if (!amrex::almostEqual(time,ctime[0])) {
156  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
157  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
158  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
159  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
160  MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
161  Rho_comp,RhoTheta_comp,1,ngvect_cons);
162  }
163 
164  if (!amrex::almostEqual(time,ftime[1])) {
165  MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev ],BaseState::r0_comp,Rho_comp,1,ngvect_cons);
166  MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev ],BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
167  MultiFab::Multiply(vars_old[lev][Vars::cons], vars_old[lev][Vars::cons],
168  Rho_comp,RhoTheta_comp,1,ngvect_cons);
169  }
170  if (!amrex::almostEqual(time,ftime[0])) {
171  MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::r0_comp,Rho_comp,1,ngvect_cons);
172  MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
173  MultiFab::Multiply(vars_new[lev][Vars::cons], vars_new[lev][Vars::cons],
174  Rho_comp,RhoTheta_comp,1,ngvect_cons);
175  }
176 
177  // Set values in the cells outside the domain boundary so that we can do the Add
178  // without worrying about uninitialized values outside the domain -- these
179  // will be filled in the physbcs call
180  mf_c.setDomainBndry(1.234e20,0,2,geom[lev]); // Do both rho and (rho theta) together
181 
182  // Add rho_0 back to rho and theta_0 back to theta
183  MultiFab::Add(mf_c, new_base_state,BaseState::r0_comp,Rho_comp,1,ngvect_cons);
184  MultiFab::Add(mf_c, new_base_state,BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
185 
186  // Multiply (theta) by rho to get (rho theta)
187  MultiFab::Multiply(mf_c,mf_c,Rho_comp,RhoTheta_comp,1,ngvect_cons);
188  }
189 
190  MultiFab::Copy(*mfs_vel[Vars::cons],mf_c,0,0,mf_c.nComp(),mf_c.nGrowVect());
191 
192  // ***************************************************************************************
193 
194  if (!cons_only)
195  {
196  mapper = &face_cons_linear_interp;
197 
198  MultiFab& mf_u = *mfs_vel[Vars::xvel];
199  MultiFab& mf_v = *mfs_vel[Vars::yvel];
200  MultiFab& mf_w = *mfs_vel[Vars::zvel];
201 
202  Vector<MultiFab*> fmf_u; Vector<MultiFab*> fmf_v; Vector<MultiFab*> fmf_w;
203  Vector<MultiFab*> cmf_u; Vector<MultiFab*> cmf_v; Vector<MultiFab*> cmf_w;
204 
205  // **********************************************************************
206 
207  if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
208  fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
209  fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
210  fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
211  } else if ( amrex::almostEqual(time,ftime[1]) ) {
212  fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
213  fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
214  fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
215  } else {
216  fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
217  fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
218  fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
219  }
220  cmf_u = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
221  cmf_v = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
222  cmf_w = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
223 
224  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
225  FillPatchTwoLevels(mf_u, ngvect_vels, IntVect(0,0,0),
226  time, cmf_u, ctime, fmf_u, ftime,
227  0, 0, 1, geom[lev-1], geom[lev],
228  refRatio(lev-1), mapper, domain_bcs_type,
230 
231  FillPatchTwoLevels(mf_v, ngvect_vels, IntVect(0,0,0),
232  time, cmf_v, ctime, fmf_v, ftime,
233  0, 0, 1, geom[lev-1], geom[lev],
234  refRatio(lev-1), mapper, domain_bcs_type,
236 
237  // We put these here because these may be used in constructing omega outside the
238  // domain when fillpatching w
239  bool do_fb = true;
240  (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
241  ngvect_vels,time,BCVars::xvel_bc, do_fb);
242  (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
243  ngvect_vels,time,BCVars::yvel_bc, do_fb);
244 
245  // **********************************************************************
246 
247  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
248  FillPatchTwoLevels(mf_w, ngvect_vels, IntVect(0,0,0),
249  time, cmf_w, ctime, fmf_w, ftime,
250  0, 0, 1, geom[lev-1], geom[lev],
251  refRatio(lev-1), mapper, domain_bcs_type,
253  } // !cons_only
254 
255  // ***************************************************************************
256  // Physical bc's at domain boundary
257  // ***************************************************************************
258  int icomp_cons = 0;
259  int ncomp_cons = mfs_vel[Vars::cons]->nComp();
260 
261  bool do_fb = true;
262 
263  if (m_r2d) fill_from_bndryregs(mfs_vel,time);
264 
265  // We call this even if use_real_bcs is true because these will fill the vertical bcs
266  // Note that we call FillBoundary inside the physbcs call
267  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
268  icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
269  if (!cons_only) {
270  // Note that we need to fill u and v in the case of terrain because we will use
271  // these in the call of WFromOmega in lateral ghost cells of the fine grid
272  // (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
273  // ngvect_vels,time,BCVars::xvel_bc, do_fb);
274  // (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
275  // ngvect_vels,time,BCVars::yvel_bc, do_fb);
276  (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
277  ngvect_vels,time,BCVars::zvel_bc, do_fb);
278  }
279 }
Here is the call graph for this function:

◆ FillWeatherDataMultiFab()

void ERF::FillWeatherDataMultiFab ( const std::string &  filename,
const amrex::Geometry &  geom_weather,
const amrex::BoxArray &  nba,
const amrex::DistributionMapping &  dm,
amrex::Vector< amrex::MultiFab > &  weather_forecast_data 
)
443 {
444 
445  Vector<Real> latvec_h, lonvec_h, xvec_h, yvec_h, zvec_h;
446  Vector<Real> rho_h, uvel_h, vvel_h, wvel_h, theta_h, qv_h, qc_h, qr_h;
447 
448  ReadCustomBinaryIC(filename, latvec_h, lonvec_h,
449  xvec_h, yvec_h, zvec_h, rho_h,
450  uvel_h, vvel_h, wvel_h,
451  theta_h, qv_h, qc_h, qr_h);
452 
453  Real zmax = *std::max_element(zvec_h.begin(), zvec_h.end());
454 
455  const auto prob_lo_erf = geom[0].ProbLoArray();
456  const auto prob_hi_erf = geom[0].ProbHiArray();
457  const auto dx_erf = geom[0].CellSizeArray();
458 
459  if (prob_hi_erf[2] >= zmax) {
460  Abort("ERROR: the maximum z of the domain (" + std::to_string(prob_hi_erf[2]) +
461  ") should be less than the maximum z in the forecast data (" + std::to_string(zmax) +
462  "). Change geometry.prob_hi[2] in the inputs to be less than " + std::to_string(zmax) + "."
463  );
464  }
465 
466  if(prob_lo_erf[0] < xvec_h.front() + 4*dx_erf[0]){
467  amrex::Abort("The xlo value of the domain has to be greater than " + std::to_string(xvec_h.front() + 4*dx_erf[0]));
468  }
469  if(prob_hi_erf[0] > xvec_h.back() - 4*dx_erf[0]){
470  amrex::Abort("The xhi value of the domain has to be less than " + std::to_string(xvec_h.back() - 4*dx_erf[0]));
471  }
472  if(prob_lo_erf[1] < yvec_h.front() + 4*dx_erf[1]){
473  amrex::Abort("The ylo value of the domain has to be greater than " + std::to_string(yvec_h.front() + 4*dx_erf[1]));
474  }
475  if(prob_hi_erf[1] > yvec_h.back() - 4*dx_erf[1]){
476  amrex::Abort("The yhi value of the domain has to be less than " + std::to_string(yvec_h.back() - 4*dx_erf[1]));
477  }
478 
479  // Number of cells
480  int nx_cells = xvec_h.size()-1;
481  int ny_cells = yvec_h.size()-1;
482 
483  const amrex::Geometry& geom0 = geom[0]; // or whatever your Geometry vector is called
484  const amrex::Box& domainBox = geom0.Domain();
485  const amrex::IntVect& domainSize = domainBox.size(); // Number of cells in each direction
486  int nz_cells = domainSize[2];
487 
488 
489  int ncomp = 10;
490  int ngrow = 0;
491 
492  int n_time = 1; // or however many time slices you want
493  weather_forecast_data.resize(n_time);
494  MultiFab& weather_mf = weather_forecast_data[0];
495  weather_mf.define(nba, dm, ncomp, ngrow);
496 
497  fill_weather_data_multifab(weather_mf, geom_weather, nx_cells+1, ny_cells+1, nz_cells+1,
498  latvec_h, lonvec_h, zvec_h,
499  rho_h,uvel_h, vvel_h, wvel_h,
500  theta_h, qv_h, qc_h, qr_h);
501 
502  //PlotMultiFab(weather_mf, geom_weather, "plt_coarse_weather", MultiFabType::NC);
503 }
void fill_weather_data_multifab(MultiFab &mf, const Geometry &geom_weather, const int nx, const int ny, const int nz, const Vector< Real > &latvec_h, const Vector< Real > &lonvec_h, const Vector< Real > &zvec_h, const Vector< Real > &rho_h, const Vector< Real > &uvel_h, const Vector< Real > &vvel_h, const Vector< Real > &wvel_h, const Vector< Real > &theta_h, const Vector< Real > &qv_h, const Vector< Real > &qc_h, const Vector< Real > &qr_h)
Definition: ERF_WeatherDataInterpolation.cpp:18
Here is the call graph for this function:

◆ get_eb()

eb_ const& ERF::get_eb ( int  lev) const
inlineprivatenoexcept
1622  {
1623  AMREX_ASSERT(lev >= 0 && lev < eb.size() && eb[lev] != nullptr);
1624  return *eb[lev];
1625  }

◆ get_projection_bc()

Array< LinOpBCType, AMREX_SPACEDIM > ERF::get_projection_bc ( amrex::Orientation::Side  side) const
noexcept
18 {
19  amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> r;
20  for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
21  if (geom[0].isPeriodic(dir)) {
22  r[dir] = LinOpBCType::Periodic;
23  } else {
24  auto bc_type = domain_bc_type[Orientation(dir,side)];
25  if (bc_type == "Outflow") {
26  r[dir] = LinOpBCType::Dirichlet;
27  } else
28  {
29  r[dir] = LinOpBCType::Neumann;
30  }
31  }
32  }
33  return r;
34 }
amrex::Array< std::string, 2 *AMREX_SPACEDIM > domain_bc_type
Definition: ERF.H:982

◆ getAdvFluxReg()

AMREX_FORCE_INLINE amrex::YAFluxRegister* ERF::getAdvFluxReg ( int  lev)
inlineprivate
1408  {
1409  return advflux_reg[lev];
1410  }

◆ getCPUTime()

static amrex::Real ERF::getCPUTime ( )
inlinestaticprivate
1500  {
1501  int numCores = amrex::ParallelDescriptor::NProcs();
1502 #ifdef _OPENMP
1503  numCores = numCores * omp_get_max_threads();
1504 #endif
1505 
1506  amrex::Real T =
1507  numCores * (amrex::ParallelDescriptor::second() - startCPUTime) +
1509 
1510  return T;
1511  }
static amrex::Real previousCPUTimeUsed
Definition: ERF.H:1496
static amrex::Real startCPUTime
Definition: ERF.H:1495
@ T
Definition: ERF_IndexDefines.H:110

◆ GotoNextLine()

void ERF::GotoNextLine ( std::istream &  is)
staticprivate

Utility to skip to next line in Header file input stream.

17 {
18  constexpr std::streamsize bl_ignore_max { 100000 };
19  is.ignore(bl_ignore_max, '\n');
20 }

◆ HurricaneTracker()

void ERF::HurricaneTracker ( int  lev,
const amrex::MultiFab &  U_new,
const amrex::MultiFab &  V_new,
const amrex::MultiFab &  W_new,
const amrex::Real  velmag_threshold,
const bool  is_track_io,
amrex::TagBoxArray *  tags = nullptr 
)
472 {
473  const auto dx = geom[levc].CellSizeArray();
474  const auto prob_lo = geom[levc].ProbLoArray();
475 
476  const int ncomp = AMREX_SPACEDIM; // Number of components (3 for 3D)
477 
478  Gpu::DeviceVector<Real> d_coords(3, 0.0); // Initialize to -1
479  Real* d_coords_ptr = d_coords.data(); // Get pointer to device vector
480  Gpu::DeviceVector<int> d_found(1,0);
481  int* d_found_ptr = d_found.data();
482 
483  MultiFab mf_cc_vel(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(0,0,0));
484  average_face_to_cellcenter(mf_cc_vel,0,{AMREX_D_DECL(&U_new,&V_new,&W_new)},0);
485 
486  // Loop through MultiFab using MFIter
487  for (MFIter mfi(mf_cc_vel); mfi.isValid(); ++mfi) {
488  const Box& box = mfi.validbox(); // Get the valid box for the current MFIter
489  const Array4<const Real>& vel_arr = mf_cc_vel.const_array(mfi); // Get the array for this MFIter
490 
491  // ParallelFor loop to check velocity magnitudes on the GPU
492  amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
493  // Access velocity components using ncomp
494  Real magnitude = 0.0; // Initialize magnitude
495 
496  for (int comp = 0; comp < ncomp; ++comp) {
497  Real vel = vel_arr(i, j, k, comp); // Access the component for each (i, j, k)
498  magnitude += vel * vel; // Sum the square of the components
499  }
500 
501  magnitude = std::sqrt(magnitude)*3.6; // Calculate magnitude
502  Real x = prob_lo[0] + (i + 0.5) * dx[0];
503  Real y = prob_lo[1] + (j + 0.5) * dx[1];
504  Real z = prob_lo[2] + (k + 0.5) * dx[2];
505 
506  // Check if magnitude exceeds threshold
507  if (z < 2.0e3 && magnitude > velmag_threshold) {
508  // Use atomic operations to set found flag and store coordinates
509  Gpu::Atomic::Add(&d_found_ptr[0], 1); // Mark as found
510 
511  // Store coordinates
512  Gpu::Atomic::Add(&d_coords_ptr[0],x); // Store x index
513  Gpu::Atomic::Add(&d_coords_ptr[1],y); // Store x index
514  Gpu::Atomic::Add(&d_coords_ptr[2],z); // Store x index
515  }
516  });
517  }
518 
519  // Synchronize to ensure all threads complete their execution
520  amrex::Gpu::streamSynchronize(); // Wait for all GPU threads to finish
521 
522  Vector<int> h_found(1,0);
523  Gpu::copy(Gpu::deviceToHost, d_found.begin(), d_found.end(), h_found.begin());
524  ParallelAllReduce::Sum(h_found.data(),
525  h_found.size(),
526  ParallelContext::CommunicatorAll());
527 
528  Real eye_x, eye_y;
529  // Broadcast coordinates if found
530  if (h_found[0] > 0) {
531  Vector<Real> h_coords(3,-1e10);
532  Gpu::copy(Gpu::deviceToHost, d_coords.begin(), d_coords.end(), h_coords.begin());
533 
534  ParallelAllReduce::Sum(h_coords.data(),
535  h_coords.size(),
536  ParallelContext::CommunicatorAll());
537 
538  eye_x = h_coords[0]/h_found[0];
539  eye_y = h_coords[1]/h_found[0];
540 
541  // Data structure to hold the hurricane track for I/O
542  if (amrex::ParallelDescriptor::IOProcessor() and is_track_io) {
543  hurricane_track_xy.push_back({eye_x, eye_y});
544  }
545 
546  if(is_track_io) {
547  return;
548  }
549 
550  Real rad_tag = 3e5*std::pow(2, max_level-1-levc);
551 
552  for (MFIter mfi(*tags); mfi.isValid(); ++mfi) {
553  TagBox& tag = (*tags)[mfi];
554  auto tag_arr = tag.array(); // Get device-accessible array
555 
556  const Box& tile_box = mfi.tilebox(); // The box for this tile
557 
558  ParallelFor(tile_box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
559  // Compute cell center coordinates
560  Real x = prob_lo[0] + (i + 0.5) * dx[0];
561  Real y = prob_lo[1] + (j + 0.5) * dx[1];
562 
563  Real dist = std::sqrt((x - eye_x)*(x - eye_x) + (y - eye_y)*(y - eye_y));
564 
565  if (dist < rad_tag) {
566  tag_arr(i,j,k) = TagBox::SET;
567  }
568  });
569  }
570  }
571 }
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_track_xy
Definition: ERF.H:153

◆ ImposeBCsOnPhi()

void ERF::ImposeBCsOnPhi ( int  lev,
amrex::MultiFab &  phi,
const amrex::Box &  subdomain 
)

Impose bc's on the pressure that comes out of the solve

13 {
14  BL_PROFILE("ERF::ImposeBCsOnPhi()");
15 
16  auto const sub_lo = lbound(subdomain);
17  auto const sub_hi = ubound(subdomain);
18 
19  auto const dom_lo = lbound(geom[lev].Domain());
20  auto const dom_hi = ubound(geom[lev].Domain());
21 
22  phi.setBndry(1.e25);
23  phi.FillBoundary(geom[lev].periodicity());
24 
25  // ****************************************************************************
26  // Impose bc's on pprime
27  // ****************************************************************************
28 #ifdef _OPENMP
29 #pragma omp parallel if (Gpu::notInLaunchRegion())
30 #endif
31  for (MFIter mfi(phi,TilingIfNotGPU()); mfi.isValid(); ++mfi)
32  {
33  Array4<Real> const& pp_arr = phi.array(mfi);
34  Box const& bx = mfi.tilebox();
35  auto const bx_lo = lbound(bx);
36  auto const bx_hi = ubound(bx);
37 
38  auto bc_type_xlo = domain_bc_type[Orientation(0,Orientation::low)];
39  auto bc_type_xhi = domain_bc_type[Orientation(0,Orientation::high)];
40  auto bc_type_ylo = domain_bc_type[Orientation(1,Orientation::low)];
41  auto bc_type_yhi = domain_bc_type[Orientation(1,Orientation::high)];
42  auto bc_type_zhi = domain_bc_type[Orientation(2,Orientation::high)];
43 
44  if ( (bx_lo.x == dom_lo.x) && (bc_type_xlo == "Outflow" || bc_type_xlo == "Open") && !solverChoice.use_real_bcs) {
45  ParallelFor(makeSlab(bx,0,dom_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
46  {
47  pp_arr(i-1,j,k) = -pp_arr(i,j,k);
48  });
49  } else if (bx_lo.x == sub_lo.x) {
50  ParallelFor(makeSlab(bx,0,sub_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
51  {
52  pp_arr(i-1,j,k) = pp_arr(i,j,k);
53  });
54  }
55 
56  if ( (bx_hi.x == dom_hi.x) && (bc_type_xhi == "Outflow" || bc_type_xhi == "Open") && !solverChoice.use_real_bcs) {
57  ParallelFor(makeSlab(bx,0,dom_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
58  {
59  pp_arr(i+1,j,k) = -pp_arr(i,j,k);
60  });
61  } else if (bx_hi.x == sub_hi.x) {
62  ParallelFor(makeSlab(bx,0,sub_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
63  {
64  pp_arr(i+1,j,k) = pp_arr(i,j,k);
65  });
66  }
67 
68  if ( (bx_lo.y == dom_lo.y) && (bc_type_ylo == "Outflow" || bc_type_ylo == "Open") && !solverChoice.use_real_bcs) {
69  ParallelFor(makeSlab(bx,1,dom_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
70  {
71  pp_arr(i,j-1,k) = -pp_arr(i,j,k);
72  });
73  } else if (bx_lo.y == sub_lo.y) {
74  ParallelFor(makeSlab(bx,1,sub_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
75  {
76  pp_arr(i,j-1,k) = pp_arr(i,j,k);
77  });
78  }
79 
80  if ( (bx_hi.y == dom_hi.y) && (bc_type_yhi == "Outflow" || bc_type_yhi == "Open") && !solverChoice.use_real_bcs) {
81  ParallelFor(makeSlab(bx,1,dom_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
82  {
83  pp_arr(i,j+1,k) = -pp_arr(i,j,k);
84  });
85  } else if (bx_hi.y == sub_hi.y) {
86  ParallelFor(makeSlab(bx,1,sub_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
87  {
88  pp_arr(i,j+1,k) = pp_arr(i,j,k);
89  });
90  }
91 
92  // At low z we are always Neumann whether the box touches the bottom boundary or not
93  Box zbx(bx); zbx.grow(0,1); zbx.grow(1,1); // Grow in x-dir and y-dir because we have filled that above
94  if (bx_lo.z == sub_lo.z) {
95  ParallelFor(makeSlab(zbx,2,dom_lo.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
96  {
97  pp_arr(i,j,k-1) = pp_arr(i,j,k);
98  });
99  }
100 
101  if ( (bx_hi.z == dom_hi.z) && (bc_type_zhi == "Outflow" || bc_type_zhi == "Open") ) {
102  ParallelFor(makeSlab(bx,2,dom_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
103  {
104  pp_arr(i,j,k+1) = -pp_arr(i,j,k);
105  });
106  } else if (bx_hi.z == sub_hi.z) {
107  ParallelFor(makeSlab(bx,2,sub_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
108  {
109  pp_arr(i,j,k+1) = pp_arr(i,j,k);
110  });
111  }
112  } // mfi
113 
114  // Now overwrite with periodic fill outside domain and fine-fine fill inside
115  phi.FillBoundary(geom[lev].periodicity());
116 }

◆ init1DArrays()

void ERF::init1DArrays ( )
private

◆ init_bcs()

void ERF::init_bcs ( )
private

Initializes data structures in the ERF class that specify which boundary conditions we are implementing on each face of the domain.

This function also maps the selected boundary condition types (e.g. Outflow, Inflow, InflowOutflow, Periodic, Dirichlet, ...) to the specific implementation needed for each variable.

Stores this information in both host and device vectors so it is available for GPU kernels.

21 {
22  bool rho_read = false;
23  bool read_prim_theta = true;
24  Vector<Real> cons_dir_init(NBCVAR_max,0.0);
25  cons_dir_init[BCVars::Rho_bc_comp] = 1.0;
26  cons_dir_init[BCVars::RhoTheta_bc_comp] = -1.0;
27  auto f = [this,&rho_read,&read_prim_theta] (std::string const& bcid, Orientation ori)
28  {
29  // These are simply defaults for Dirichlet faces -- they should be over-written below
31  m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = -1.0; // It is important to set this negative
32  // because the sign is tested on below
33  for (int n = BCVars::RhoKE_bc_comp; n < BCVars::xvel_bc; n++) {
34  m_bc_extdir_vals[n][ori] = 0.0;
35  }
36 
37  m_bc_extdir_vals[BCVars::xvel_bc][ori] = 0.0; // default
40 
41  // These are simply defaults for Neumann gradients -- they should be over-written below
44 
53 
57 
58  std::string pp_text = pp_prefix + "." + bcid;
59  ParmParse pp(pp_text);
60 
61  std::string bc_type_in;
62  if (pp.query("type", bc_type_in) <= 0)
63  {
64  pp_text = bcid;
65  pp = ParmParse(pp_text);
66  pp.query("type", bc_type_in);
67  }
68 
69  std::string bc_type = amrex::toLower(bc_type_in);
70 
71  if (bc_type == "symmetry")
72  {
73  // Print() << bcid << " set to symmetry.\n";
75  domain_bc_type[ori] = "Symmetry";
76  }
77  else if (bc_type == "outflow")
78  {
79  // Print() << bcid << " set to outflow.\n";
81  domain_bc_type[ori] = "Outflow";
82  }
83  else if (bc_type == "open")
84  {
85  // Print() << bcid << " set to open.\n";
86  AMREX_ASSERT_WITH_MESSAGE((ori.coordDir() != 2), "Open boundary not valid on zlo or zhi!");
88  domain_bc_type[ori] = "Open";
89  }
90  else if (bc_type == "ho_outflow")
91  {
93  domain_bc_type[ori] = "HO_Outflow";
94  }
95 
96  else if (bc_type == "inflow" || bc_type == "inflow_outflow")
97  {
98  if (bc_type == "inflow") {
99  // Print() << bcid << " set to inflow.\n";
101  domain_bc_type[ori] = "Inflow";
102  } else {
103  // Print() << bcid << " set to inflow_outflow.\n";
105  domain_bc_type[ori] = "InflowOutflow";
106  }
107 
108  std::vector<Real> v;
109  if (input_bndry_planes && m_r2d->ingested_velocity()) {
113  } else {
114  // Test for input data file if at xlo face
115  std::string dirichlet_file;
116  auto file_exists = pp.query("dirichlet_file", dirichlet_file);
117  if (file_exists) {
118  pp.query("read_prim_theta", read_prim_theta);
119  init_Dirichlet_bc_data(dirichlet_file);
120  } else {
121  pp.getarr("velocity", v, 0, AMREX_SPACEDIM);
122  m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
123  m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
124  m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
125  }
126  }
127 
128  Real rho_in = 0.;
129  if (input_bndry_planes && m_r2d->ingested_density()) {
131  } else {
132  if (!pp.query("density", rho_in)) {
133  amrex::Print() << "Using interior values to set conserved vars" << std::endl;
134  }
135  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
136  }
137 
138  bool th_read = (th_bc_data[0].data()!=nullptr);
139  Real theta_in = 0.;
140  if (input_bndry_planes && m_r2d->ingested_theta()) {
142  } else if (!th_read) {
143  if (rho_in > 0) {
144  pp.get("theta", theta_in);
145  }
146  m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = rho_in*theta_in;
147  }
148 
149  Real scalar_in = 0.;
150  if (input_bndry_planes && m_r2d->ingested_scalar()) {
152  } else {
153  if (pp.query("scalar", scalar_in))
154  m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = rho_in*scalar_in;
155  }
156 
157  if (solverChoice.moisture_type != MoistureType::None) {
158  Real qv_in = 0.;
159  if (input_bndry_planes && m_r2d->ingested_q1()) {
161  } else {
162  if (pp.query("qv", qv_in))
163  m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = rho_in*qv_in;
164  }
165  Real qc_in = 0.;
166  if (input_bndry_planes && m_r2d->ingested_q2()) {
168  } else {
169  if (pp.query("qc", qc_in))
170  m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = rho_in*qc_in;
171  }
172  }
173 
174  Real KE_in = 0.;
175  if (input_bndry_planes && m_r2d->ingested_KE()) {
177  } else {
178  if (pp.query("KE", KE_in))
179  m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = rho_in*KE_in;
180  }
181  }
182  else if (bc_type == "noslipwall")
183  {
184  // Print() << bcid <<" set to no-slip wall.\n";
186  domain_bc_type[ori] = "NoSlipWall";
187 
188  std::vector<Real> v;
189 
190  // The values of m_bc_extdir_vals default to 0.
191  // But if we find "velocity" in the inputs file, use those values instead.
192  if (pp.queryarr("velocity", v, 0, AMREX_SPACEDIM))
193  {
194  v[ori.coordDir()] = 0.0;
195  m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
196  m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
197  m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
198  }
199 
200  Real rho_in;
201  rho_read = pp.query("density", rho_in);
202  if (rho_read)
203  {
204  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
205  }
206 
207  Real theta_in;
208  if (pp.query("theta", theta_in))
209  {
211  }
212 
213  Real theta_grad_in;
214  if (pp.query("theta_grad", theta_grad_in))
215  {
216  m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
217  }
218 
219  Real qv_in;
220  if (pp.query("qv", qv_in))
221  {
223  }
224  }
225  else if (bc_type == "slipwall")
226  {
227  // Print() << bcid <<" set to slip wall.\n";
228 
230  domain_bc_type[ori] = "SlipWall";
231 
232  Real rho_in;
233  rho_read = pp.query("density", rho_in);
234  if (rho_read)
235  {
236  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
237  }
238 
239  Real theta_in;
240  if (pp.query("theta", theta_in))
241  {
243  }
244 
245  Real rho_grad_in;
246  if (pp.query("density_grad", rho_grad_in))
247  {
248  m_bc_neumann_vals[BCVars::Rho_bc_comp][ori] = rho_grad_in;
249  }
250 
251  Real theta_grad_in;
252  if (pp.query("theta_grad", theta_grad_in))
253  {
254  m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
255  }
256  }
257  else if (bc_type == "surface_layer")
258  {
260  domain_bc_type[ori] = "surface_layer";
261  }
262  else
263  {
265  }
266 
267  if (geom[0].isPeriodic(ori.coordDir())) {
268  domain_bc_type[ori] = "Periodic";
269  if (phys_bc_type[ori] == ERF_BC::undefined)
270  {
272  } else {
273  Abort("Wrong BC type for periodic boundary");
274  }
275  }
276 
277  if (phys_bc_type[ori] == ERF_BC::undefined)
278  {
279  Print() << "BC Type specified for face " << bcid << " is " << bc_type_in << std::endl;
280  Abort("This BC type is unknown");
281  }
282  };
283 
284  f("xlo", Orientation(Direction::x,Orientation::low));
285  f("xhi", Orientation(Direction::x,Orientation::high));
286  f("ylo", Orientation(Direction::y,Orientation::low));
287  f("yhi", Orientation(Direction::y,Orientation::high));
288  f("zlo", Orientation(Direction::z,Orientation::low));
289  f("zhi", Orientation(Direction::z,Orientation::high));
290 
291  // *****************************************************************************
292  //
293  // Here we translate the physical boundary conditions -- one type per face --
294  // into logical boundary conditions for each velocity component
295  //
296  // *****************************************************************************
297  {
298  domain_bcs_type.resize(AMREX_SPACEDIM+NBCVAR_max);
299  domain_bcs_type_d.resize(AMREX_SPACEDIM+NBCVAR_max);
300 
301  for (OrientationIter oit; oit; ++oit) {
302  Orientation ori = oit();
303  int dir = ori.coordDir();
304  Orientation::Side side = ori.faceDir();
305  auto const bct = phys_bc_type[ori];
306  if ( bct == ERF_BC::symmetry )
307  {
308  if (side == Orientation::low) {
309  for (int i = 0; i < AMREX_SPACEDIM; i++) {
311  }
313  } else {
314  for (int i = 0; i < AMREX_SPACEDIM; i++) {
316  }
318  }
319  }
320  else if (bct == ERF_BC::outflow or bct == ERF_BC::ho_outflow )
321  {
322  if (side == Orientation::low) {
323  for (int i = 0; i < AMREX_SPACEDIM; i++) {
325  }
326  if (!solverChoice.anelastic[0]) {
328  }
329  } else {
330  for (int i = 0; i < AMREX_SPACEDIM; i++) {
332  }
333  if (!solverChoice.anelastic[0]) {
335  }
336  }
337  }
338  else if (bct == ERF_BC::open)
339  {
340  if (side == Orientation::low) {
341  for (int i = 0; i < AMREX_SPACEDIM; i++)
343  } else {
344  for (int i = 0; i < AMREX_SPACEDIM; i++)
346  }
347  }
348  else if (bct == ERF_BC::inflow)
349  {
350  if (side == Orientation::low) {
351  for (int i = 0; i < AMREX_SPACEDIM; i++) {
353  if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
355  }
356  }
357  } else {
358  for (int i = 0; i < AMREX_SPACEDIM; i++) {
360  if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
362  }
363  }
364  }
365  }
366  else if (bct == ERF_BC::inflow_outflow)
367  {
368  if (side == Orientation::low) {
369  for (int i = 0; i < AMREX_SPACEDIM; i++) {
371  }
372  } else {
373  for (int i = 0; i < AMREX_SPACEDIM; i++) {
375  }
376  }
377  }
378  else if (bct == ERF_BC::no_slip_wall)
379  {
380  if (side == Orientation::low) {
381  for (int i = 0; i < AMREX_SPACEDIM; i++) {
383  }
384  } else {
385  for (int i = 0; i < AMREX_SPACEDIM; i++) {
387  }
388  }
389  }
390  else if (bct == ERF_BC::slip_wall)
391  {
392  if (side == Orientation::low) {
393  for (int i = 0; i < AMREX_SPACEDIM; i++) {
395  }
396  // Only normal direction has ext_dir
398 
399  } else {
400  for (int i = 0; i < AMREX_SPACEDIM; i++) {
402  }
403  // Only normal direction has ext_dir
405  }
406  }
407  else if (bct == ERF_BC::periodic)
408  {
409  if (side == Orientation::low) {
410  for (int i = 0; i < AMREX_SPACEDIM; i++) {
412  }
413  } else {
414  for (int i = 0; i < AMREX_SPACEDIM; i++) {
416  }
417  }
418  }
419  else if ( bct == ERF_BC::surface_layer )
420  {
421  AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
425  }
426  }
427  }
428 
429  // *****************************************************************************
430  //
431  // Here we translate the physical boundary conditions -- one type per face --
432  // into logical boundary conditions for each cell-centered variable
433  // (including the base state variables)
434  // NOTE: all "scalars" share the same type of boundary condition
435  //
436  // *****************************************************************************
437  {
438  for (OrientationIter oit; oit; ++oit) {
439  Orientation ori = oit();
440  int dir = ori.coordDir();
441  Orientation::Side side = ori.faceDir();
442  auto const bct = phys_bc_type[ori];
443  if ( bct == ERF_BC::symmetry )
444  {
445  if (side == Orientation::low) {
446  for (int i = 0; i < NBCVAR_max; i++) {
448  }
449  } else {
450  for (int i = 0; i < NBCVAR_max; i++) {
452  }
453  }
454  }
455  else if ( bct == ERF_BC::outflow )
456  {
457  if (side == Orientation::low) {
458  for (int i = 0; i < NBCVAR_max; i++) {
460  }
461  } else {
462  for (int i = 0; i < NBCVAR_max; i++) {
464  }
465  }
466  }
467  else if ( bct == ERF_BC::ho_outflow )
468  {
469  if (side == Orientation::low) {
470  for (int i = 0; i < NBCVAR_max; i++) {
472  }
473  } else {
474  for (int i = 0; i < NBCVAR_max; i++) {
476  }
477  }
478  }
479  else if ( bct == ERF_BC::open )
480  {
481  if (side == Orientation::low) {
482  for (int i = 0; i < NBCVAR_max; i++)
484  } else {
485  for (int i = 0; i < NBCVAR_max; i++)
487  }
488  }
489  else if ( bct == ERF_BC::no_slip_wall )
490  {
491  if (side == Orientation::low) {
492  for (int i = 0; i < NBCVAR_max; i++) {
494  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
495  if (rho_read) {
497  } else {
499  }
500  }
501  }
502  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
504  }
505  } else {
506  for (int i = 0; i < NBCVAR_max; i++) {
508  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
509  if (rho_read) {
511  } else {
513  }
514  }
515  }
516  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
518  }
519  }
520  }
521  else if (bct == ERF_BC::slip_wall)
522  {
523  if (side == Orientation::low) {
524  for (int i = 0; i < NBCVAR_max; i++) {
526  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
527  if (rho_read) {
529  } else {
531  }
532  }
533  }
534  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
536  }
537  if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
539  }
540  } else {
541  for (int i = 0; i < NBCVAR_max; i++) {
543  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
544  if (rho_read) {
546  } else {
548  }
549  }
550  }
551  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
553  }
554  if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
556  }
557  }
558  }
559  else if (bct == ERF_BC::inflow)
560  {
561  if (side == Orientation::low) {
562  for (int i = 0; i < NBCVAR_max; i++) {
564  if ((BCVars::cons_bc+i == RhoTheta_comp) &&
565  (th_bc_data[0].data() != nullptr))
566  {
567  if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
568  }
569  else if (input_bndry_planes && dir < 2 && (
570  ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp) && m_r2d->ingested_density()) ||
571  ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp) && m_r2d->ingested_theta() ) ||
572  ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp) && m_r2d->ingested_KE() ) ||
573  ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
574  ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp) && m_r2d->ingested_q1() ) ||
575  ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp) && m_r2d->ingested_q2() )) )
576  {
578  }
579  else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
581  }
582  }
583  } else {
584  for (int i = 0; i < NBCVAR_max; i++) {
586  if ((BCVars::cons_bc+i == RhoTheta_comp) &&
587  (th_bc_data[0].data() != nullptr))
588  {
589  if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
590  }
591  else if (input_bndry_planes && dir < 2 && (
592  ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp) && m_r2d->ingested_density()) ||
593  ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp) && m_r2d->ingested_theta() ) ||
594  ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp) && m_r2d->ingested_KE() ) ||
595  ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
596  ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp) && m_r2d->ingested_q1() ) ||
597  ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp) && m_r2d->ingested_q2() )
598  ) )
599  {
601  }
602  else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
604  }
605  }
606  }
607  }
608  else if (bct == ERF_BC::inflow_outflow )
609  {
610  if (side == Orientation::low) {
611  for (int i = 0; i < NBCVAR_max; i++) {
613  if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
615  }
616  }
617  } else {
618  for (int i = 0; i < NBCVAR_max; i++) {
620  if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
622  }
623  }
624  }
625  }
626  else if (bct == ERF_BC::periodic)
627  {
628  if (side == Orientation::low) {
629  for (int i = 0; i < NBCVAR_max; i++) {
631  }
632  } else {
633  for (int i = 0; i < NBCVAR_max; i++) {
635  }
636  }
637  }
638  else if ( bct == ERF_BC::surface_layer )
639  {
640  AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
641  for (int i = 0; i < NBCVAR_max; i++) {
643  }
644  }
645  }
646  }
647 
648  // NOTE: Gpu:copy is a wrapper to htod_memcpy (GPU) or memcpy (CPU) and is a blocking comm
649  Gpu::copy(Gpu::hostToDevice, domain_bcs_type.begin(), domain_bcs_type.end(), domain_bcs_type_d.begin());
650 }
#define NBCVAR_max
Definition: ERF_IndexDefines.H:29
@ ho_outflow
@ inflow_outflow
void init_Dirichlet_bc_data(const std::string input_file)
Definition: ERF_InitBCs.cpp:652
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max > m_bc_neumann_vals
Definition: ERF.H:988
@ RhoQ6_bc_comp
Definition: ERF_IndexDefines.H:86
@ RhoQ1_bc_comp
Definition: ERF_IndexDefines.H:81
@ RhoQ4_bc_comp
Definition: ERF_IndexDefines.H:84
@ RhoKE_bc_comp
Definition: ERF_IndexDefines.H:79
@ RhoQ3_bc_comp
Definition: ERF_IndexDefines.H:83
@ RhoTheta_bc_comp
Definition: ERF_IndexDefines.H:78
@ RhoQ2_bc_comp
Definition: ERF_IndexDefines.H:82
@ Rho_bc_comp
Definition: ERF_IndexDefines.H:77
@ RhoQ5_bc_comp
Definition: ERF_IndexDefines.H:85
@ neumann
Definition: ERF_IndexDefines.H:213
@ open
Definition: ERF_IndexDefines.H:215
@ reflect_odd
Definition: ERF_IndexDefines.H:205
@ hoextrap
Definition: ERF_IndexDefines.H:216
@ foextrap
Definition: ERF_IndexDefines.H:208
@ ext_dir
Definition: ERF_IndexDefines.H:209
@ ext_dir_prim
Definition: ERF_IndexDefines.H:211
@ ext_dir_upwind
Definition: ERF_IndexDefines.H:217
@ int_dir
Definition: ERF_IndexDefines.H:206
@ neumann_int
Definition: ERF_IndexDefines.H:214
@ reflect_even
Definition: ERF_IndexDefines.H:207
Here is the call graph for this function:

◆ init_custom()

void ERF::init_custom ( int  lev)
private

Wrapper for custom problem-specific initialization routines that can be defined by the user as they set up a new problem in ERF. This wrapper handles all the overhead of defining the perturbation as well as initializing the random seed if needed.

This wrapper calls a user function to customize initialization on a per-Fab level inside an MFIter loop, so all the MultiFab operations are hidden from the user.

Parameters
levInteger specifying the current level
27 {
28  auto& lev_new = vars_new[lev];
29 
30  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
31  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
32 
33  MultiFab cons_pert(lev_new[Vars::cons].boxArray(), lev_new[Vars::cons].DistributionMap(),
34  lev_new[Vars::cons].nComp() , lev_new[Vars::cons].nGrow());
35  MultiFab xvel_pert(lev_new[Vars::xvel].boxArray(), lev_new[Vars::xvel].DistributionMap(), 1, lev_new[Vars::xvel].nGrowVect());
36  MultiFab yvel_pert(lev_new[Vars::yvel].boxArray(), lev_new[Vars::yvel].DistributionMap(), 1, lev_new[Vars::yvel].nGrowVect());
37  MultiFab zvel_pert(lev_new[Vars::zvel].boxArray(), lev_new[Vars::zvel].DistributionMap(), 1, lev_new[Vars::zvel].nGrowVect());
38 
39  // Default all perturbations to zero
40  cons_pert.setVal(0.);
41  xvel_pert.setVal(0.);
42  yvel_pert.setVal(0.);
43  zvel_pert.setVal(0.);
44 
45 #ifdef _OPENMP
46 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
47 #endif
48  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
49  {
50  const Box &bx = mfi.tilebox();
51  const Box &xbx = mfi.tilebox(IntVect(1,0,0));
52  const Box &ybx = mfi.tilebox(IntVect(0,1,0));
53  const Box &zbx = mfi.tilebox(IntVect(0,0,1));
54 
55  const auto &cons_pert_arr = cons_pert.array(mfi);
56  const auto &xvel_pert_arr = xvel_pert.array(mfi);
57  const auto &yvel_pert_arr = yvel_pert.array(mfi);
58  const auto &zvel_pert_arr = zvel_pert.array(mfi);
59 
60  Array4<Real const> cons_arr = lev_new[Vars::cons].const_array(mfi);
61  Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
62  Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
63 
64  // Here we arbitrarily choose the x-oriented map factor -- this should be generalized
65  Array4<Real const> mf_m = mapfac[lev][MapFacType::m_x]->const_array(mfi);
66  Array4<Real const> mf_u = mapfac[lev][MapFacType::u_x]->const_array(mfi);
67  Array4<Real const> mf_v = mapfac[lev][MapFacType::v_y]->const_array(mfi);
68 
69  Array4<Real> r_hse_arr = r_hse.array(mfi);
70  Array4<Real> p_hse_arr = p_hse.array(mfi);
71 
72  prob->init_custom_pert(bx, xbx, ybx, zbx, cons_arr, cons_pert_arr,
73  xvel_pert_arr, yvel_pert_arr, zvel_pert_arr,
74  r_hse_arr, p_hse_arr, z_nd_arr, z_cc_arr,
75  geom[lev].data(), mf_m, mf_u, mf_v,
76  solverChoice);
77  } //mfi
78 
79  // Add problem-specific perturbation to background flow if not doing anelastic with fixed-in-time density
81  MultiFab::Add(lev_new[Vars::cons], cons_pert, Rho_comp, Rho_comp, 1, cons_pert.nGrow());
82  }
83  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoTheta_comp, RhoTheta_comp, 1, cons_pert.nGrow());
84  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoScalar_comp,RhoScalar_comp,NSCALARS, cons_pert.nGrow());
85 
86  // RhoKE is relevant if using Deardorff with LES, k-equation for RANS, or MYNN with PBL
87  if (solverChoice.turbChoice[lev].use_tke) {
88  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoKE_comp, RhoKE_comp, 1, cons_pert.nGrow());
89  }
90 
91  if (solverChoice.moisture_type != MoistureType::None) {
92  int qstate_size = micro->Get_Qstate_Size();
93  for (int q_offset(0); q_offset<qstate_size; ++q_offset) {
94  int q_idx = RhoQ1_comp+q_offset;
95  MultiFab::Add(lev_new[Vars::cons], cons_pert, q_idx, q_idx, 1, cons_pert.nGrow());
96  }
97  }
98 
99  MultiFab::Add(lev_new[Vars::xvel], xvel_pert, 0, 0, 1, xvel_pert.nGrowVect());
100  MultiFab::Add(lev_new[Vars::yvel], yvel_pert, 0, 0, 1, yvel_pert.nGrowVect());
101  MultiFab::Add(lev_new[Vars::zvel], zvel_pert, 0, 0, 1, zvel_pert.nGrowVect());
102 }
const Box xbx
Definition: ERF_SetupDiff.H:7
const Box ybx
Definition: ERF_SetupDiff.H:8
bool fixed_density
Definition: ERF_DataStruct.H:943
Here is the call graph for this function:

◆ init_Dirichlet_bc_data()

void ERF::init_Dirichlet_bc_data ( const std::string  input_file)
private
653 {
654  // Read the dirichlet_input file
655  Print() << "dirichlet_input file location : " << input_file << std::endl;
656  std::ifstream input_reader(input_file);
657  if (!input_reader.is_open()) {
658  amrex::Abort("Error opening the dirichlet_input file.\n");
659  }
660 
661  Print() << "Successfully opened the dirichlet_input file. Now reading... " << std::endl;
662  std::string line;
663 
664  // Size of Ninp (number of z points in input file)
665  Vector<Real> z_inp_tmp, u_inp_tmp, v_inp_tmp, w_inp_tmp, th_inp_tmp;
666 
667  // Top and bot for domain
668  const int klo = geom[0].Domain().smallEnd()[2];
669  const int khi = geom[0].Domain().bigEnd()[2];
670  const Real zbot = zlevels_stag[0][klo];
671  const Real ztop = zlevels_stag[0][khi+1];
672 
673  // Flag if theta input
674  Real th_init = -300.0;
675  bool th_read{false};
676 
677  // Add surface
678  z_inp_tmp.push_back(zbot); // height above sea level [m]
679  u_inp_tmp.push_back(0.);
680  v_inp_tmp.push_back(0.);
681  w_inp_tmp.push_back(0.);
682  th_inp_tmp.push_back(th_init);
683 
684  // Read the vertical profile at each given height
685  Real z, u, v, w, th;
686  while(std::getline(input_reader, line)) {
687  std::istringstream iss_z(line);
688 
689  Vector<Real> rval_v;
690  Real rval;
691  while (iss_z >> rval) {
692  rval_v.push_back(rval);
693  }
694  z = rval_v[0];
695  u = rval_v[1];
696  v = rval_v[2];
697  w = rval_v[3];
698 
699  // Format without theta
700  if (rval_v.size() == 4) {
701  if (z == zbot) {
702  u_inp_tmp[0] = u;
703  v_inp_tmp[0] = v;
704  w_inp_tmp[0] = w;
705  } else {
706  AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
707  z_inp_tmp.push_back(z);
708  u_inp_tmp.push_back(u);
709  v_inp_tmp.push_back(v);
710  w_inp_tmp.push_back(w);
711  if (z >= ztop) break;
712  }
713  } else if (rval_v.size() == 5) {
714  th_read = true;
715  th = rval_v[4];
716  if (z == zbot) {
717  u_inp_tmp[0] = u;
718  v_inp_tmp[0] = v;
719  w_inp_tmp[0] = w;
720  th_inp_tmp[0] = th;
721  } else {
722  AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
723  z_inp_tmp.push_back(z);
724  u_inp_tmp.push_back(u);
725  v_inp_tmp.push_back(v);
726  w_inp_tmp.push_back(w);
727  th_inp_tmp.push_back(th);
728  if (z >= ztop) break;
729  }
730  } else {
731  Abort("Unknown inflow file format!");
732  }
733  }
734 
735  // Ensure we set a reasonable theta surface
736  if (th_read) {
737  if (th_inp_tmp[0] == th_init) {
738  Real slope = (th_inp_tmp[2] - th_inp_tmp[1]) / (z_inp_tmp[2] - z_inp_tmp[1]);
739  Real dz = z_inp_tmp[0] - z_inp_tmp[1];
740  th_inp_tmp[0] = slope * dz + th_inp_tmp[1];
741  }
742  }
743 
744  amrex::Print() << "Successfully read and interpolated the dirichlet_input file..." << std::endl;
745  input_reader.close();
746 
747  for (int lev = 0; lev <= max_level; lev++) {
748 
749  const int Nz = geom[lev].Domain().size()[2];
750 
751  // Size of Nz (domain grid)
752  Vector<Real> zcc_inp(Nz );
753  Vector<Real> znd_inp(Nz+1);
754  Vector<Real> u_inp(Nz ); xvel_bc_data[lev].resize(Nz ,0.0);
755  Vector<Real> v_inp(Nz ); yvel_bc_data[lev].resize(Nz ,0.0);
756  Vector<Real> w_inp(Nz+1); zvel_bc_data[lev].resize(Nz+1,0.0);
757  Vector<Real> th_inp;
758  if (th_read) {
759  th_inp.resize(Nz);
760  th_bc_data[lev].resize(Nz, 0.0);
761  }
762 
763  // At this point, we have an input from zbot up to
764  // z_inp_tmp[N-1] >= ztop. Now, interpolate to grid level 0 heights
765  const int Ninp = z_inp_tmp.size();
766  for (int k(0); k<Nz; ++k) {
767  zcc_inp[k] = 0.5 * (zlevels_stag[lev][k] + zlevels_stag[lev][k+1]);
768  znd_inp[k] = zlevels_stag[lev][k+1];
769  u_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), u_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
770  v_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), v_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
771  w_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), znd_inp[k], Ninp);
772  if (th_read) {
773  th_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), th_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
774  }
775  }
776  znd_inp[Nz] = ztop;
777  w_inp[Nz] = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), ztop, Ninp);
778 
779  // Copy host data to the device
780  Gpu::copy(Gpu::hostToDevice, u_inp.begin(), u_inp.end(), xvel_bc_data[lev].begin());
781  Gpu::copy(Gpu::hostToDevice, v_inp.begin(), v_inp.end(), yvel_bc_data[lev].begin());
782  Gpu::copy(Gpu::hostToDevice, w_inp.begin(), w_inp.end(), zvel_bc_data[lev].begin());
783  if (th_read) {
784  Gpu::copy(Gpu::hostToDevice, th_inp.begin(), th_inp.end(), th_bc_data[lev].begin());
785  }
786 
787  // NOTE: These device vectors are passed to the PhysBC constructors when that
788  // class is instantiated in ERF_MakeNewArrays.cpp.
789  } // lev
790 }
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real interpolate_1d(const amrex::Real *alpha, const amrex::Real *beta, const amrex::Real alpha_interp, const int alpha_size)
Definition: ERF_Interpolation_1D.H:12
Here is the call graph for this function:

◆ init_from_hse()

void ERF::init_from_hse ( int  lev)

Initialize the background flow to have the calculated HSE density and rho*theta calculated from the HSE pressure. In general, the hydrostatically balanced density and pressure (r_hse and p_hse from base_state) used here may be calculated through a solver path such as:

ERF::initHSE(lev)

  • call prob->erf_init_dens_hse(...)
    • call Problem::init_isentropic_hse(...), to simultaneously calculate r_hse and p_hse with Newton iteration – assuming constant theta
    • save r_hse
  • call ERF::enforce_hse(...), calculates p_hse from saved r_hse (redundant, but needed because p_hse is not necessarily calculated by the Problem implementation) and pi_hse and th_hse – note: this pressure does not exactly match the p_hse from before because what is calculated by init_isentropic_hse comes from the EOS whereas what is calculated here comes from the hydro- static equation
Parameters
levInteger specifying the current level
33 {
34  auto& lev_new = vars_new[lev];
35 
36  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
37  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
38 
39 #ifdef _OPENMP
40 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
41 #endif
42  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
43  {
44  const Box &gbx = mfi.growntilebox(1);
45  const Array4<Real >& cons_arr = lev_new[Vars::cons].array(mfi);
46  const Array4<Real const>& r_hse_arr = r_hse.const_array(mfi);
47  const Array4<Real const>& p_hse_arr = p_hse.const_array(mfi);
48 
49  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
50  {
51  cons_arr(i,j,k,Rho_comp) = r_hse_arr(i,j,k);
52  cons_arr(i,j,k,RhoTheta_comp) = getRhoThetagivenP(p_hse_arr(i,j,k));
53  });
54  } //mfi
55 }
Here is the call graph for this function:

◆ init_from_input_sounding()

void ERF::init_from_input_sounding ( int  lev)

High level wrapper for initializing scalar and velocity level data from input sounding data.

Parameters
levInteger specifying the current level
53 {
54  // We only want to read the file once -- here we fill one FArrayBox (per variable) that spans the domain
55  if (lev == 0) {
57  Error("input_sounding file name must be provided via input");
58  }
59 
61 
62  // this will interpolate the input profiles to the nominal height levels
63  // (ranging from 0 to the domain top)
64  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
66  }
67 
68  // this will calculate the hydrostatically balanced density and pressure
69  // profiles following WRF ideal.exe
70  if (solverChoice.sounding_type == SoundingType::Ideal) {
72  } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
73  solverChoice.sounding_type == SoundingType::DryIsentropic) {
74  input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
76  }
77 
78  } else {
79  //
80  // We need to do this interp from coarse level in order to set the values of
81  // the base state inside the domain but outside of the fine region
82  //
83  base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
84  //
85  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
86  // have been pre-filled - this includes ghost cells both inside and outside
87  // the domain
88  //
89  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
90  IntVect(0,0,0), // do not fill ghost cells outside the domain
91  base_state[lev-1], 0, 0, base_state[lev].nComp(),
92  geom[lev-1], geom[lev],
93  refRatio(lev-1), &cell_cons_interp,
95 
96  // We need to do this here because the interpolation above may leave corners unfilled
97  // when the corners need to be filled by, for example, reflection of the fine ghost
98  // cell outside the fine region but inide the domain.
99  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
100  }
101 
102  auto& lev_new = vars_new[lev];
103 
104  // updated if sounding is ideal (following WRF) or isentropic
105  const bool l_isentropic = (solverChoice.sounding_type == SoundingType::Isentropic ||
106  solverChoice.sounding_type == SoundingType::DryIsentropic);
107  const bool sounding_ideal_or_isentropic = (solverChoice.sounding_type == SoundingType::Ideal ||
108  l_isentropic);
109  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
110  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
111  MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
112  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
113  MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
114 
115  const Real l_gravity = solverChoice.gravity;
116  const Real l_rdOcp = solverChoice.rdOcp;
117  const bool l_moist = (solverChoice.moisture_type != MoistureType::None);
118 
119 #ifdef _OPENMP
120 #pragma omp parallel if (Gpu::notInLaunchRegion())
121 #endif
122  for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
123  const Box &bx = mfi.tilebox();
124  const auto &cons_arr = lev_new[Vars::cons].array(mfi);
125  const auto &xvel_arr = lev_new[Vars::xvel].array(mfi);
126  const auto &yvel_arr = lev_new[Vars::yvel].array(mfi);
127  const auto &zvel_arr = lev_new[Vars::zvel].array(mfi);
128  Array4<Real> r_hse_arr = r_hse.array(mfi);
129  Array4<Real> p_hse_arr = p_hse.array(mfi);
130  Array4<Real> pi_hse_arr = pi_hse.array(mfi);
131  Array4<Real> th_hse_arr = th_hse.array(mfi);
132  Array4<Real> qv_hse_arr = qv_hse.array(mfi);
133 
134  Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
135  Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
136 
137  if (sounding_ideal_or_isentropic)
138  {
139  // HSE will be initialized here, interpolated from values previously
140  // calculated by calc_rho_p or calc_rho_p_isentropic
142  bx, cons_arr,
143  r_hse_arr, p_hse_arr, pi_hse_arr, th_hse_arr, qv_hse_arr,
144  geom[lev].data(), z_cc_arr,
145  l_gravity, l_rdOcp, l_moist, input_sounding_data,
146  l_isentropic);
147  }
148  else
149  {
150  // This assumes rho_0 = 1.0
151  // HSE will be calculated later with call to initHSE
153  bx, cons_arr,
154  geom[lev].data(), z_cc_arr,
155  l_moist, input_sounding_data);
156  }
157 
159  bx, xvel_arr, yvel_arr, zvel_arr,
160  geom[lev].data(), z_nd_arr,
162 
163  } //mfi
164 }
void init_bx_scalars_from_input_sounding(const Box &bx, Array4< Real > const &state, GeometryData const &geomdata, Array4< const Real > const &z_cc_arr, const bool &l_moist, InputSoundingData const &inputSoundingData)
Definition: ERF_InitFromInputSounding.cpp:176
void init_bx_scalars_from_input_sounding_hse(const Box &bx, Array4< Real > const &state, Array4< Real > const &r_hse_arr, Array4< Real > const &p_hse_arr, Array4< Real > const &pi_hse_arr, Array4< Real > const &th_hse_arr, Array4< Real > const &qv_hse_arr, GeometryData const &geomdata, Array4< const Real > const &z_cc_arr, const Real &l_gravity, const Real &l_rdOcp, const bool &l_moist, InputSoundingData const &inputSoundingData, const bool &l_isentropic)
Definition: ERF_InitFromInputSounding.cpp:238
void init_bx_velocities_from_input_sounding(const Box &bx, Array4< Real > const &x_vel, Array4< Real > const &y_vel, Array4< Real > const &z_vel, GeometryData const &geomdata, Array4< const Real > const &z_nd_arr, InputSoundingData const &inputSoundingData)
Definition: ERF_InitFromInputSounding.cpp:374
InputSoundingData input_sounding_data
Definition: ERF.H:761
@ rho0_bc_comp
Definition: ERF_IndexDefines.H:98
@ qv0_comp
Definition: ERF_IndexDefines.H:67
void resize_arrays()
Definition: ERF_InputSoundingData.H:60
int n_sounding_files
Definition: ERF_InputSoundingData.H:395
void read_from_file(const amrex::Geometry &geom, const amrex::Vector< amrex::Real > &zlevels_stag, int itime)
Definition: ERF_InputSoundingData.H:77
amrex::Vector< std::string > input_sounding_file
Definition: ERF_InputSoundingData.H:393
void calc_rho_p(int itime)
Definition: ERF_InputSoundingData.H:173
void calc_rho_p_isentropic(int itime)
Definition: ERF_InputSoundingData.H:259
bool assume_dry
Definition: ERF_InputSoundingData.H:398
static SoundingType sounding_type
Definition: ERF_DataStruct.H:898
Here is the call graph for this function:

◆ init_geo_wind_profile()

void ERF::init_geo_wind_profile ( const std::string  input_file,
amrex::Vector< amrex::Real > &  u_geos,
amrex::Gpu::DeviceVector< amrex::Real > &  u_geos_d,
amrex::Vector< amrex::Real > &  v_geos,
amrex::Gpu::DeviceVector< amrex::Real > &  v_geos_d,
const amrex::Geometry &  lgeom,
const amrex::Vector< amrex::Real > &  zlev_stag 
)
private
17 {
18  const int klo = 0;
19  const int khi = lgeom.Domain().bigEnd()[AMREX_SPACEDIM-1];
20  const amrex::Real dz = lgeom.CellSize()[AMREX_SPACEDIM-1];
21 
22  const bool grid_stretch = (zlev_stag.size() > 0);
23  const Real zbot = (grid_stretch) ? zlev_stag[klo] : lgeom.ProbLo(AMREX_SPACEDIM-1);
24  const Real ztop = (grid_stretch) ? zlev_stag[khi+1] : lgeom.ProbHi(AMREX_SPACEDIM-1);
25 
26  amrex::Print() << "Reading geostrophic wind profile from " << input_file << std::endl;
27  std::ifstream profile_reader(input_file);
28  if(!profile_reader.is_open()) {
29  amrex::Error("Error opening the abl_geo_wind_table\n");
30  }
31 
32  // First, read the input data into temp vectors
33  std::string line;
34  Vector<Real> z_inp, Ug_inp, Vg_inp;
35  Real z, Ug, Vg;
36  amrex::Print() << "z Ug Vg" << std::endl;
37  while(std::getline(profile_reader, line)) {
38  std::istringstream iss(line);
39  iss >> z >> Ug >> Vg;
40  amrex::Print() << z << " " << Ug << " " << Vg << std::endl;
41  z_inp.push_back(z);
42  Ug_inp.push_back(Ug);
43  Vg_inp.push_back(Vg);
44  if (z >= ztop) break;
45  }
46 
47  const int Ninp = z_inp.size();
48  AMREX_ALWAYS_ASSERT(z_inp[0] <= zbot);
49  AMREX_ALWAYS_ASSERT(z_inp[Ninp-1] >= ztop);
50 
51  // Now, interpolate vectors to the cell centers
52  for (int k = 0; k <= khi; k++) {
53  z = (grid_stretch) ? 0.5 * (zlev_stag[k] + zlev_stag[k+1])
54  : zbot + (k + 0.5) * dz;
55  u_geos[k] = interpolate_1d(z_inp.dataPtr(), Ug_inp.dataPtr(), z, Ninp);
56  v_geos[k] = interpolate_1d(z_inp.dataPtr(), Vg_inp.dataPtr(), z, Ninp);
57  }
58 
59  // Copy from host version to device version
60  Gpu::copy(Gpu::hostToDevice, u_geos.begin(), u_geos.end(), u_geos_d.begin());
61  Gpu::copy(Gpu::hostToDevice, v_geos.begin(), v_geos.end(), v_geos_d.begin());
62 
63  profile_reader.close();
64 }
Here is the call graph for this function:

◆ init_immersed_forcing()

void ERF::init_immersed_forcing ( int  lev)

Set velocities in cells that are immersed to be 0 (or a very small number)

Parameters
levInteger specifying the current level
16 {
17  auto& lev_new = vars_new[lev];
18  MultiFab* terrain_blank = terrain_blanking[lev].get();
19 
20 #ifdef _OPENMP
21 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
22 #endif
23  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
24  {
25  const Box &xbx = mfi.tilebox(IntVect(1,0,0));
26  const Box &ybx = mfi.tilebox(IntVect(0,1,0));
27  const Box &zbx = mfi.tilebox(IntVect(0,0,1));
28  const Real epsilon = 1e-2;
29 
30  const Array4<const Real>& t_blank_arr = terrain_blank->const_array(mfi);
31 
32  const auto &xvel_arr = lev_new[Vars::xvel].array(mfi);
33  const auto &yvel_arr = lev_new[Vars::yvel].array(mfi);
34  const auto &zvel_arr = lev_new[Vars::zvel].array(mfi);
35 
36  // Set the x,y,z-velocities
37  ParallelFor(xbx, ybx, zbx,
38  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
39  const Real t_blank = 0.5 * (t_blank_arr(i, j, k) + t_blank_arr(i-1, j, k));
40  if (t_blank >= 0.9) { xvel_arr(i, j, k) = epsilon; }
41  },
42  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
43  const Real t_blank = 0.5 * (t_blank_arr(i, j, k) + t_blank_arr(i, j-1, k));
44  if (t_blank >= 0.9) { yvel_arr(i, j, k) = epsilon; }
45  },
46  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
47  const Real t_blank = 0.5 * (t_blank_arr(i, j, k) + t_blank_arr(i, j, k-1));
48  if (t_blank >= 0.9) { zvel_arr(i, j, k) = epsilon; }
49  });
50  } //mfi
51 }
real(c_double), parameter epsilon
Definition: ERF_module_model_constants.F90:12
Here is the call graph for this function:

◆ init_only()

void ERF::init_only ( int  lev,
amrex::Real  time 
)
1932 {
1933  t_new[lev] = time;
1934  t_old[lev] = time - 1.e200;
1935 
1936  auto& lev_new = vars_new[lev];
1937  auto& lev_old = vars_old[lev];
1938 
1939  // Loop over grids at this level to initialize our grid data
1940  lev_new[Vars::cons].setVal(0.0); lev_old[Vars::cons].setVal(0.0);
1941  lev_new[Vars::xvel].setVal(0.0); lev_old[Vars::xvel].setVal(0.0);
1942  lev_new[Vars::yvel].setVal(0.0); lev_old[Vars::yvel].setVal(0.0);
1943  lev_new[Vars::zvel].setVal(0.0); lev_old[Vars::zvel].setVal(0.0);
1944 
1945  // Initialize background flow (optional)
1946  if (solverChoice.init_type == InitType::Input_Sounding) {
1947  // The physbc's need the terrain but are needed for initHSE
1948  // We have already made the terrain in the call to init_zphys
1949  // in MakeNewLevelFromScratch
1950  make_physbcs(lev);
1951 
1952  // Now init the base state and the data itself
1954 
1955  // The base state has been initialized by integrating vertically
1956  // through the sounding for ideal (like WRF) or isentropic approaches
1957  if (solverChoice.sounding_type == SoundingType::Ideal ||
1958  solverChoice.sounding_type == SoundingType::Isentropic ||
1959  solverChoice.sounding_type == SoundingType::DryIsentropic) {
1960  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(solverChoice.use_gravity,
1961  "Gravity should be on to be consistent with sounding initialization.");
1962  } else { // SoundingType::ConstantDensity
1963  AMREX_ASSERT_WITH_MESSAGE(!solverChoice.use_gravity,
1964  "Constant density probably doesn't make sense with gravity");
1965  initHSE();
1966  }
1967 
1968 #ifdef ERF_USE_NETCDF
1969  }
1970  else if (solverChoice.init_type == InitType::WRFInput)
1971  {
1972  // The base state is initialized from WRF wrfinput data, output by
1973  // ideal.exe or real.exe
1974 
1975  init_from_wrfinput(lev, *mf_C1H, *mf_C2H, *mf_MUB, *mf_PSFC[lev]);
1976 
1977  if (lev==0) {
1978  if ((start_time > 0) && (start_time != start_bdy_time)) {
1979  Print() << "Ignoring specified start_time="
1980  << std::setprecision(timeprecision) << start_time
1981  << std::endl;
1982  }
1983  }
1984 
1985  start_time = start_bdy_time;
1986 
1987  use_datetime = true;
1988 
1989  // The physbc's need the terrain but are needed for initHSE
1990  if (!solverChoice.use_real_bcs) {
1991  make_physbcs(lev);
1992  }
1993  }
1994  else if (solverChoice.init_type == InitType::NCFile)
1995  {
1996  // The state is initialized by reading from a Netcdf file
1997  init_from_ncfile(lev);
1998 
1999  // The physbc's need the terrain but are needed for initHSE
2000  make_physbcs(lev);
2001  }
2002  else if (solverChoice.init_type == InitType::Metgrid)
2003  {
2004  // The base state is initialized from data output by WPS metgrid;
2005  // we will rebalance after interpolation
2006  init_from_metgrid(lev);
2007 #endif
2008  } else if (solverChoice.init_type == InitType::Uniform) {
2009  // Initialize a uniform background field and base state based on the
2010  // problem-specified reference density and temperature
2011 
2012  // The physbc's need the terrain but are needed for initHSE
2013  make_physbcs(lev);
2014 
2015  init_uniform(lev);
2016  initHSE(lev);
2017  } else {
2018  // No background flow initialization specified, initialize the
2019  // background field to be equal to the base state, calculated from the
2020  // problem-specific erf_init_dens_hse
2021 
2022  // The bc's need the terrain but are needed for initHSE
2023  make_physbcs(lev);
2024 
2025  // We will initialize the state from the background state so must set that first
2026  initHSE(lev);
2027  init_from_hse(lev);
2028  }
2029 
2030  // Add problem-specific flow features
2031  //
2032  // Notes:
2033  // - This calls init_custom_pert that is defined for each problem
2034  // - This may modify the base state
2035  // - The fields set by init_custom_pert are **perturbations** to the
2036  // background flow set based on init_type
2037  if (solverChoice.init_type != InitType::NCFile) {
2038  init_custom(lev);
2039  }
2040 
2041  // Ensure that the face-based data are the same on both sides of a periodic domain.
2042  // The data associated with the lower grid ID is considered the correct value.
2043  lev_new[Vars::xvel].OverrideSync(geom[lev].periodicity());
2044  lev_new[Vars::yvel].OverrideSync(geom[lev].periodicity());
2045  lev_new[Vars::zvel].OverrideSync(geom[lev].periodicity());
2046 
2047  if(solverChoice.spongeChoice.sponge_type == "input_sponge"){
2048  input_sponge(lev);
2049  }
2050 
2051  // Initialize turbulent perturbation
2052  if (solverChoice.pert_type == PerturbationType::Source ||
2053  solverChoice.pert_type == PerturbationType::Direct ||
2054  solverChoice.pert_type == PerturbationType::CPM) {
2055  turbPert_update(lev, 0.);
2056  turbPert_amplitude(lev);
2057  }
2058 
2059  // Set initial velocity field for immersed cells to be close to 0
2060  if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
2061  init_immersed_forcing(lev);
2062  }
2063 }
const int timeprecision
Definition: ERF.H:1023
void init_from_input_sounding(int lev)
Definition: ERF_InitFromInputSounding.cpp:52
std::unique_ptr< amrex::MultiFab > mf_MUB
Definition: ERF.H:1248
std::unique_ptr< amrex::MultiFab > mf_C2H
Definition: ERF.H:1247
void init_custom(int lev)
Definition: ERF_InitCustom.cpp:26
void init_from_hse(int lev)
Definition: ERF_InitFromHSE.cpp:32
void initHSE()
Initialize HSE.
Definition: ERF_Init1D.cpp:146
void turbPert_update(const int lev, const amrex::Real dt)
Definition: ERF_InitTurbPert.cpp:12
void input_sponge(int lev)
Definition: ERF_InitSponge.cpp:17
void make_physbcs(int lev)
Definition: ERF_MakeNewArrays.cpp:752
void init_immersed_forcing(int lev)
Definition: ERF_InitImmersedForcing.cpp:15
void init_uniform(int lev)
Definition: ERF_InitUniform.cpp:17
std::unique_ptr< amrex::MultiFab > mf_C1H
Definition: ERF.H:1246
void turbPert_amplitude(const int lev)
Definition: ERF_InitTurbPert.cpp:32
bool use_gravity
Definition: ERF_DataStruct.H:955

◆ init_stuff()

void ERF::init_stuff ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm,
amrex::Vector< amrex::MultiFab > &  lev_new,
amrex::Vector< amrex::MultiFab > &  lev_old,
amrex::MultiFab &  tmp_base_state,
std::unique_ptr< amrex::MultiFab > &  tmp_zphys_nd 
)
private
28 {
29  // ********************************************************************************************
30  // Base state holds r_0, pres_0, pi_0, th_0 (in that order)
31  //
32  // Here is where we set the number of ghost cells for the base state!
33  // ********************************************************************************************
34  int ngb = (solverChoice.terrain_type == TerrainType::EB) ? 4 : 3;
35  tmp_base_state.define(ba,dm,BaseState::num_comps,ngb);
36  tmp_base_state.setVal(0.);
37 
38  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
39  base_state_new[lev].define(ba,dm,BaseState::num_comps,base_state[lev].nGrowVect());
40  base_state_new[lev].setVal(0.);
41  }
42 
43  // ********************************************************************************************
44  // Allocate terrain arrays
45  // ********************************************************************************************
46 
47  BoxArray ba_nd(ba);
48  ba_nd.surroundingNodes();
49 
50  // NOTE: this is where we actually allocate z_phys_nd -- but here it's called "tmp_zphys_nd"
51  // We need this to be one greater than the ghost cells to handle levels > 0
52 
53  int ngrow = ComputeGhostCells(solverChoice) + 2;
54  tmp_zphys_nd = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
55 
56  z_phys_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
57  init_default_zphys(lev, geom[lev], *tmp_zphys_nd, *z_phys_cc[lev]);
58 
59  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh)
60  {
61  detJ_cc_new[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
62  detJ_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
63 
64  ax_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
65  ay_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
66  az_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
67 
68  z_t_rk[lev] = std::make_unique<MultiFab>( convert(ba, IntVect(0,0,1)), dm, 1, 1 );
69 
70  z_phys_nd_new[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
71  z_phys_nd_src[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
72  z_phys_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
73  }
74  else
75  {
76  z_phys_nd_new[lev] = nullptr;
77  detJ_cc_new[lev] = nullptr;
78 
79  z_phys_nd_src[lev] = nullptr;
80  z_phys_cc_src[lev] = nullptr;
81  detJ_cc_src[lev] = nullptr;
82 
83  z_t_rk[lev] = nullptr;
84  }
85 
86  if (SolverChoice::terrain_type == TerrainType::ImmersedForcing)
87  {
88  terrain_blanking[lev] = std::make_unique<MultiFab>(ba,dm,1,ngrow);
89  terrain_blanking[lev]->setVal(1.0);
90  }
91 
92  // We use these area arrays regardless of terrain, EB or none of the above
93  detJ_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
94  ax[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
95  ay[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
96  az[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
97 
98  detJ_cc[lev]->setVal(1.0);
99  ax[lev]->setVal(1.0);
100  ay[lev]->setVal(1.0);
101  az[lev]->setVal(1.0);
102 
103  // ********************************************************************************************
104  // Create wall distance array for RANS modeling
105  // ********************************************************************************************
106  if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
107  walldist[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
108  walldist[lev]->setVal(1e23);
109  } else {
110  walldist[lev] = nullptr;
111  }
112 
113  // ********************************************************************************************
114  // These are the persistent containers for the old and new data
115  // ********************************************************************************************
116  int ncomp;
117  if (lev > 0) {
118  ncomp = vars_new[lev-1][Vars::cons].nComp();
119  } else {
120  int n_qstate = micro->Get_Qstate_Size();
121  ncomp = NDRY + NSCALARS + n_qstate;
122  }
123 
124  // ********************************************************************************************
125  // The number of ghost cells for density must be 1 greater than that for velocity
126  // so that we can go back in forth between velocity and momentum on all faces
127  // ********************************************************************************************
128  int ngrow_state = ComputeGhostCells(solverChoice) + 1;
129  int ngrow_vels = ComputeGhostCells(solverChoice);
130 
131  // ********************************************************************************************
132  // New solution data containers
133  // ********************************************************************************************
134  if (solverChoice.terrain_type != TerrainType::EB) {
135  lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state);
136  lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state);
137  } else {
138  // EB: Define the MultiFabs with the EBFactory
139  lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state, MFInfo(), EBFactory(lev));
140  lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state, MFInfo(), EBFactory(lev));
141  }
142  lev_new[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
143  lev_old[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
144 
145  lev_new[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
146  lev_old[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
147 
148  gradp[lev][GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp[lev][GpVars::gpx].setVal(0.);
149  gradp[lev][GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp[lev][GpVars::gpy].setVal(0.);
150  gradp[lev][GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp[lev][GpVars::gpz].setVal(0.);
151 
152  // Note that we need the ghost cells in the z-direction if we are doing any
153  // kind of domain decomposition in the vertical (at level 0 or above)
154  lev_new[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
155  lev_old[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
156 
158  pp_inc[lev].define(ba, dm, 1, 1);
159  pp_inc[lev].setVal(0.0);
160  }
161 
162  // We use this in the fast substepping only
163  if (solverChoice.anelastic[lev] == 0) {
164  lagged_delta_rt[lev].define(ba, dm, 1, 1);
165  lagged_delta_rt[lev].setVal(0.0);
166  }
167 
168  // We use these for advecting the slow variables, whether anelastic or compressible
169  avg_xmom[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, 1);
170  avg_ymom[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, 1);
171  avg_zmom[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, 1);
172  avg_xmom[lev].setVal(0.0); avg_ymom[lev].setVal(0.0); avg_zmom[lev].setVal(0.0);
173 
174  // ********************************************************************************************
175  // These are just used for scratch in the time integrator but we might as well define them here
176  // ********************************************************************************************
177  if (solverChoice.terrain_type != TerrainType::EB) {
178  rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
179  rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
180 
181  rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
182  rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
183 
184  rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
185  rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
186  } else {
187  // EB: Define the MultiFabs with the EBFactory
188  rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
189  rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
190 
191  rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
192  rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
193 
194  rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
195  rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
196  }
197 
198  if (lev > 0) {
199  //xmom_crse_rhs[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, IntVect{0});
200  //ymom_crse_rhs[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, IntVect{0});
201  zmom_crse_rhs[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, IntVect{0});
202  }
203 
204  // We do this here just so they won't be undefined in the initial FillPatch
205  rU_old[lev].setVal(1.2e21);
206  rV_old[lev].setVal(3.4e22);
207  rW_old[lev].setVal(5.6e23);
208  rU_new[lev].setVal(1.2e21);
209  rV_new[lev].setVal(3.4e22);
210  rW_new[lev].setVal(5.6e23);
211 
212  // ********************************************************************************************
213  // These are just time averaged fields for diagnostics
214  // ********************************************************************************************
215 
216  // NOTE: We are not completing a fillpach call on the time averaged data;
217  // which would copy on intersection and interpolate from coarse.
218  // Therefore, we are restarting the averaging when the ba changes,
219  // this may give poor statistics for dynamic mesh refinement.
220  vel_t_avg[lev] = nullptr;
222  vel_t_avg[lev] = std::make_unique<MultiFab>(ba, dm, 4, 0); // Each vel comp and the mag
223  vel_t_avg[lev]->setVal(0.0);
224  t_avg_cnt[lev] = 0.0;
225  }
226 
227  // ********************************************************************************************
228  // Initialize flux registers whenever we create/re-create a level
229  // ********************************************************************************************
230  if (solverChoice.coupling_type == CouplingType::TwoWay) {
231  if (lev == 0) {
232  advflux_reg[0] = nullptr;
233  } else {
234  int ncomp_reflux = vars_new[0][Vars::cons].nComp();
235  advflux_reg[lev] = new YAFluxRegister(ba , grids[lev-1],
236  dm , dmap[lev-1],
237  geom[lev], geom[lev-1],
238  ref_ratio[lev-1], lev, ncomp_reflux);
239  }
240  }
241 
242  // ********************************************************************************************
243  // Define Theta_prim storage if using surface_layer BC
244  // ********************************************************************************************
245  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
246  Theta_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
247  if (solverChoice.moisture_type != MoistureType::None) {
248  Qv_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
249  Qr_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
250  } else {
251  Qv_prim[lev] = nullptr;
252  Qr_prim[lev] = nullptr;
253  }
254  } else {
255  Theta_prim[lev] = nullptr;
256  Qv_prim[lev] = nullptr;
257  Qr_prim[lev] = nullptr;
258  }
259 
260  // ********************************************************************************************
261  // Map factors
262  // ********************************************************************************************
263  BoxList bl2d_mf = ba.boxList();
264  for (auto& b : bl2d_mf) {
265  b.setRange(2,0);
266  }
267  BoxArray ba2d_mf(std::move(bl2d_mf));
268 
269  mapfac[lev].resize(MapFacType::num);
270  mapfac[lev][MapFacType::m_x] = std::make_unique<MultiFab>( ba2d_mf,dm,1,IntVect(3,3,0));
271  mapfac[lev][MapFacType::u_x] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(1,0,0)),dm,1,IntVect(3,3,0));
272  mapfac[lev][MapFacType::v_x] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(0,1,0)),dm,1,IntVect(3,3,0));
273 
274 #if 0
275  // For now we comment this out to avoid CI failures but we will need to re-enable
276  // this if using non-conformal mappings
278  mapfac[lev][MapFacType::m_y] = std::make_unique<MultiFab>(ba2d_mf,dm,1,IntVect(3,3,0));
279  }
281  mapfac[lev][MapFacType::u_y] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(1,0,0)),dm,1,IntVect(3,3,0));
282  }
284  mapfac[lev][MapFacType::v_y] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(0,1,0)),dm,1,IntVect(3,3,0));
285  }
286 #endif
287 
289  for (int i = 0; i < 3; i++) {
290  mapfac[lev][i]->setVal(0.5);
291  }
292  for (int i = 3; i < mapfac[lev].size(); i++) {
293  mapfac[lev][i]->setVal(0.25);
294  }
295  } else {
296  for (int i = 0; i < mapfac[lev].size(); i++) {
297  mapfac[lev][i]->setVal(1.0);
298  }
299  }
300 
301  // ********************************************************************************************
302  // Build 1D BA and 2D BA
303  // ********************************************************************************************
304  BoxList bl1d = ba.boxList();
305  for (auto& b : bl1d) {
306  b.setRange(0,0);
307  b.setRange(1,0);
308  }
309  ba1d[lev] = BoxArray(std::move(bl1d));
310 
311  // Build 2D BA
312  BoxList bl2d = ba.boxList();
313  for (auto& b : bl2d) {
314  b.setRange(2,0);
315  }
316  ba2d[lev] = BoxArray(std::move(bl2d));
317 
318  IntVect ng = vars_new[lev][Vars::cons].nGrowVect();
319 
320  if (lev == 0) {
321  mf_C1H = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
322  mf_C2H = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
323  mf_MUB = std::make_unique<MultiFab>(ba2d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
324  }
325 
326  mf_PSFC[lev] = std::make_unique<MultiFab>(ba2d[lev],dm,1,ng);
327 
328  //*********************************************************
329  // Variables for Fitch model for windfarm parametrization
330  //*********************************************************
331 #if defined(ERF_USE_WINDFARM)
332  if (solverChoice.windfarm_type == WindFarmType::Fitch){
333  vars_windfarm[lev].define(ba, dm, 5, ngrow_state); // V, dVabsdt, dudt, dvdt, dTKEdt
334  }
335  if (solverChoice.windfarm_type == WindFarmType::EWP){
336  vars_windfarm[lev].define(ba, dm, 3, ngrow_state); // dudt, dvdt, dTKEdt
337  }
338  if (solverChoice.windfarm_type == WindFarmType::SimpleAD) {
339  vars_windfarm[lev].define(ba, dm, 2, ngrow_state);// dudt, dvdt
340  }
341  if (solverChoice.windfarm_type == WindFarmType::GeneralAD) {
342  vars_windfarm[lev].define(ba, dm, 3, ngrow_state);// dudt, dvdt, dwdt
343  }
344  Nturb[lev].define(ba, dm, 1, ngrow_state); // Number of turbines in a cell
345  SMark[lev].define(ba, dm, 2, 1); // Free stream velocity/source term
346  // sampling marker in a cell - 2 components
347 #endif
348 
349 
350 #ifdef ERF_USE_WW3_COUPLING
351  // create a new BoxArray and DistributionMapping for a MultiFab with 1 box
352  BoxArray ba_onegrid(geom[lev].Domain());
353  BoxList bl2d_onegrid = ba_onegrid.boxList();
354  for (auto& b : bl2d_onegrid) {
355  b.setRange(2,0);
356  }
357  BoxArray ba2d_onegrid(std::move(bl2d_onegrid));
358  Vector<int> pmap;
359  pmap.resize(1);
360  pmap[0]=0;
361  DistributionMapping dm_onegrid(ba2d_onegrid);
362  dm_onegrid.define(pmap);
363 
364  Hwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
365  Lwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
366 
367  BoxList bl2d_wave = ba.boxList();
368  for (auto& b : bl2d_wave) {
369  b.setRange(2,0);
370  }
371  BoxArray ba2d_wave(std::move(bl2d_wave));
372 
373  Hwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
374  Lwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
375 
376  std::cout<<ba_onegrid<<std::endl;
377  std::cout<<ba2d_onegrid<<std::endl;
378  std::cout<<dm_onegrid<<std::endl;
379 #endif
380 
381 
382  //*********************************************************
383  // Radiation heating source terms
384  //*********************************************************
385  if (solverChoice.rad_type != RadiationType::None)
386  {
387  qheating_rates[lev] = std::make_unique<MultiFab>(ba, dm, 2, 0);
388  rad_fluxes[lev] = std::make_unique<MultiFab>(ba, dm, 4, 0);
389  qheating_rates[lev]->setVal(0.);
390  rad_fluxes[lev]->setVal(0.);
391  }
392 
393  //*********************************************************
394  // Radiation fluxes for coupling to LSM
395  //*********************************************************
396 
397  // NOTE: Finer levels do not need to coincide with the bottom domain boundary
398  // at k=0. We make slabs here with the kmin for a given box. Therefore,
399  // care must be taken before applying these fluxes to an LSM model. For
400 
401  // Radiative fluxes for LSM
402  if (solverChoice.lsm_type != LandSurfaceType::None &&
403  solverChoice.rad_type != RadiationType::None)
404  {
405  BoxList m_bl = ba.boxList();
406  for (auto& b : m_bl) {
407  int kmin = b.smallEnd(2);
408  b.setRange(2,kmin);
409  }
410  BoxArray m_ba(std::move(m_bl));
411 
412  sw_lw_fluxes[lev] = std::make_unique<MultiFab>(m_ba, dm, 6, 0); // DIR/DIF VIS/NIR (4), NET SW (1), LW (1)
413  solar_zenith[lev] = std::make_unique<MultiFab>(m_ba, dm, 1, 0);
414 
415  sw_lw_fluxes[lev]->setVal(0.);
416  solar_zenith[lev]->setVal(0.);
417  }
418 
419  //*********************************************************
420  // Turbulent perturbation region initialization
421  //*********************************************************
422  if (solverChoice.pert_type == PerturbationType::Source ||
423  solverChoice.pert_type == PerturbationType::Direct ||
424  solverChoice.pert_type == PerturbationType::CPM)
425  {
426  amrex::Box bnd_bx = ba.minimalBox();
428  turbPert.init_tpi(lev, bnd_bx.smallEnd(), bnd_bx.bigEnd(), geom[lev].CellSizeArray(),
429  ba, dm, ngrow_state, pp_prefix, refRatio(), max_level);
430  }
431 
432  //
433  // Define the land mask here and set it to all land by default
434  // NOTE: the logic below will BREAK if we have any grids not touching the bottom boundary
435  //
436  {
437  lmask_lev[lev].resize(1);
438  auto ngv = lev_new[Vars::cons].nGrowVect(); ngv[2] = 0;
439  BoxList bl2d_mask = ba.boxList();
440  for (auto& b : bl2d_mask) {
441  b.setRange(2,0);
442  }
443  BoxArray ba2d_mask(std::move(bl2d_mask));
444  lmask_lev[lev][0] = std::make_unique<iMultiFab>(ba2d_mask,dm,1,ngv);
445  lmask_lev[lev][0]->setVal(1);
446  lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
447 
448  land_type_lev[lev].resize(1);
449  land_type_lev[lev][0] = std::make_unique<iMultiFab>(ba2d_mask,dm,1,ngv);
450  land_type_lev[lev][0]->setVal(0);
451  land_type_lev[lev][0]->FillBoundary(geom[lev].periodicity());
452 
453  soil_type_lev[lev].resize(1);
454  soil_type_lev[lev][0] = std::make_unique<iMultiFab>(ba2d_mask,dm,1,ngv);
455  soil_type_lev[lev][0]->setVal(0);
456  soil_type_lev[lev][0]->FillBoundary(geom[lev].periodicity());
457 
458  urb_frac_lev[lev].resize(1);
459  urb_frac_lev[lev][0] = std::make_unique<MultiFab>(ba2d_mask,dm,1,ngv);
460  urb_frac_lev[lev][0]->setVal(1.0);
461  urb_frac_lev[lev][0]->FillBoundary(geom[lev].periodicity());
462  }
463 
464  // Read in tables needed for windfarm simulations
465  // fill in Nturb multifab - number of turbines in each mesh cell
466  // write out the vtk files for wind turbine location and/or
467  // actuator disks
468  #ifdef ERF_USE_WINDFARM
469  //init_windfarm(lev);
470  #endif
471 }
@ num
Definition: ERF_DataStruct.H:23
#define NDRY
Definition: ERF_IndexDefines.H:13
void init_default_zphys(int, const Geometry &geom, MultiFab &z_phys_nd, MultiFab &z_phys_cc)
Definition: ERF_TerrainMetrics.cpp:15
static AMREX_FORCE_INLINE int ComputeGhostCells(const SolverChoice &sc)
Definition: ERF.H:1349
amrex::EBFArrayBoxFactory const & EBFactory(int lev) const noexcept
Definition: ERF.H:1628
@ num_comps
Definition: ERF_IndexDefines.H:68
@ gpz
Definition: ERF_IndexDefines.H:152
@ gpy
Definition: ERF_IndexDefines.H:151
@ gpx
Definition: ERF_IndexDefines.H:150
bool test_mapfactor
Definition: ERF_DataStruct.H:948
void init_tpi_type(const PerturbationType &pert_type)
Definition: ERF_TurbPertStruct.H:28
void init_tpi(const int lev, const amrex::IntVect &valid_box_lo, const amrex::IntVect &valid_box_hi, const amrex::GpuArray< amrex::Real, 3 > dx, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, const int ngrow_state, std::string pp_prefix, const amrex::Vector< amrex::IntVect > refRatio, const int max_level)
Definition: ERF_TurbPertStruct.H:45
Here is the call graph for this function:

◆ init_thin_body()

void ERF::init_thin_body ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
751 {
752  //********************************************************************************************
753  // Thin immersed body
754  // *******************************************************************************************
755 #if 0
756  if ((solverChoice.advChoice.zero_xflux.size() > 0) ||
757  (solverChoice.advChoice.zero_yflux.size() > 0) ||
758  (solverChoice.advChoice.zero_zflux.size() > 0))
759  {
760  overset_imask[lev] = std::make_unique<iMultiFab>(ba,dm,1,0);
761  overset_imask[lev]->setVal(1); // == value is unknown (to be solved)
762  }
763 #endif
764 
765  if (solverChoice.advChoice.zero_xflux.size() > 0) {
766  amrex::Print() << "Setting up thin immersed body for "
767  << solverChoice.advChoice.zero_xflux.size() << " xfaces" << std::endl;
768  BoxArray ba_xf(ba);
769  ba_xf.surroundingNodes(0);
770  thin_xforce[lev] = std::make_unique<MultiFab>(ba_xf,dm,1,0);
771  thin_xforce[lev]->setVal(0.0);
772  xflux_imask[lev] = std::make_unique<iMultiFab>(ba_xf,dm,1,0);
773  xflux_imask[lev]->setVal(1);
774  for ( MFIter mfi(*xflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
775  {
776  Array4<int> const& imask_arr = xflux_imask[lev]->array(mfi);
777  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
778  Box xbx = mfi.nodaltilebox(0);
779  for (int iv=0; iv < solverChoice.advChoice.zero_xflux.size(); ++iv) {
780  const auto& faceidx = solverChoice.advChoice.zero_xflux[iv];
781  if ((faceidx[0] >= xbx.smallEnd(0)) && (faceidx[0] <= xbx.bigEnd(0)) &&
782  (faceidx[1] >= xbx.smallEnd(1)) && (faceidx[1] <= xbx.bigEnd(1)) &&
783  (faceidx[2] >= xbx.smallEnd(2)) && (faceidx[2] <= xbx.bigEnd(2)))
784  {
785  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
786  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
787  //imask_cell_arr(faceidx[0]-1,faceidx[1],faceidx[2]) = 0;
788  amrex::AllPrint() << " mask xface at " << faceidx << std::endl;
789  }
790  }
791  }
792  } else {
793  thin_xforce[lev] = nullptr;
794  xflux_imask[lev] = nullptr;
795  }
796 
797  if (solverChoice.advChoice.zero_yflux.size() > 0) {
798  amrex::Print() << "Setting up thin immersed body for "
799  << solverChoice.advChoice.zero_yflux.size() << " yfaces" << std::endl;
800  BoxArray ba_yf(ba);
801  ba_yf.surroundingNodes(1);
802  thin_yforce[lev] = std::make_unique<MultiFab>(ba_yf,dm,1,0);
803  thin_yforce[lev]->setVal(0.0);
804  yflux_imask[lev] = std::make_unique<iMultiFab>(ba_yf,dm,1,0);
805  yflux_imask[lev]->setVal(1);
806  for ( MFIter mfi(*yflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
807  {
808  Array4<int> const& imask_arr = yflux_imask[lev]->array(mfi);
809  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
810  Box ybx = mfi.nodaltilebox(1);
811  for (int iv=0; iv < solverChoice.advChoice.zero_yflux.size(); ++iv) {
812  const auto& faceidx = solverChoice.advChoice.zero_yflux[iv];
813  if ((faceidx[0] >= ybx.smallEnd(0)) && (faceidx[0] <= ybx.bigEnd(0)) &&
814  (faceidx[1] >= ybx.smallEnd(1)) && (faceidx[1] <= ybx.bigEnd(1)) &&
815  (faceidx[2] >= ybx.smallEnd(2)) && (faceidx[2] <= ybx.bigEnd(2)))
816  {
817  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
818  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
819  //imask_cell_arr(faceidx[0],faceidx[1]-1,faceidx[2]) = 0;
820  amrex::AllPrint() << " mask yface at " << faceidx << std::endl;
821  }
822  }
823  }
824  } else {
825  thin_yforce[lev] = nullptr;
826  yflux_imask[lev] = nullptr;
827  }
828 
829  if (solverChoice.advChoice.zero_zflux.size() > 0) {
830  amrex::Print() << "Setting up thin immersed body for "
831  << solverChoice.advChoice.zero_zflux.size() << " zfaces" << std::endl;
832  BoxArray ba_zf(ba);
833  ba_zf.surroundingNodes(2);
834  thin_zforce[lev] = std::make_unique<MultiFab>(ba_zf,dm,1,0);
835  thin_zforce[lev]->setVal(0.0);
836  zflux_imask[lev] = std::make_unique<iMultiFab>(ba_zf,dm,1,0);
837  zflux_imask[lev]->setVal(1);
838  for ( MFIter mfi(*zflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
839  {
840  Array4<int> const& imask_arr = zflux_imask[lev]->array(mfi);
841  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
842  Box zbx = mfi.nodaltilebox(2);
843  for (int iv=0; iv < solverChoice.advChoice.zero_zflux.size(); ++iv) {
844  const auto& faceidx = solverChoice.advChoice.zero_zflux[iv];
845  if ((faceidx[0] >= zbx.smallEnd(0)) && (faceidx[0] <= zbx.bigEnd(0)) &&
846  (faceidx[1] >= zbx.smallEnd(1)) && (faceidx[1] <= zbx.bigEnd(1)) &&
847  (faceidx[2] >= zbx.smallEnd(2)) && (faceidx[2] <= zbx.bigEnd(2)))
848  {
849  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
850  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
851  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]-1) = 0;
852  amrex::AllPrint() << " mask zface at " << faceidx << std::endl;
853  }
854  }
855  }
856  } else {
857  thin_zforce[lev] = nullptr;
858  zflux_imask[lev] = nullptr;
859  }
860 }
amrex::Vector< amrex::IntVect > zero_yflux
Definition: ERF_AdvStruct.H:438
amrex::Vector< amrex::IntVect > zero_xflux
Definition: ERF_AdvStruct.H:437
amrex::Vector< amrex::IntVect > zero_zflux
Definition: ERF_AdvStruct.H:439

◆ init_uniform()

void ERF::init_uniform ( int  lev)
private

Use problem-specific reference density and temperature to set the background state to a uniform value.

Parameters
levInteger specifying the current level
18 {
19  auto& lev_new = vars_new[lev];
20  for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
21  const Box &gbx = mfi.growntilebox(1);
22  const auto &cons_arr = lev_new[Vars::cons].array(mfi);
23  prob->init_uniform(gbx, cons_arr);
24  }
25 }

◆ init_zphys()

void ERF::init_zphys ( int  lev,
amrex::Real  time 
)
600 {
601  if (solverChoice.init_type != InitType::WRFInput && solverChoice.init_type != InitType::Metgrid)
602  {
603  if (lev > 0) {
604  //
605  // First interpolate from coarser level if there is one
606  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
607  // have been pre-filled - this includes ghost cells both inside and outside
608  // the domain
609  //
610  InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
611  IntVect(0,0,0), // do not fill ghost cells outside the domain
612  *z_phys_nd[lev-1], 0, 0, 1,
613  geom[lev-1], geom[lev],
614  refRatio(lev-1), &node_bilinear_interp,
616  }
617 
618  int ngrow = ComputeGhostCells(solverChoice) + 2;
619  Box bx(surroundingNodes(Geom(lev).Domain())); bx.grow(ngrow);
620  FArrayBox terrain_fab(makeSlab(bx,2,0),1);
621 
622  //
623  // If we are using fitted mesh then we use the surface as defined above
624  // If we are not using fitted mesh but are using z_levels, we still need z_phys (for now)
625  // but we need to use a flat terrain for the mesh itself (the EB data has already been made
626  // from the correct terrain)
627  //
628  if (solverChoice.terrain_type != TerrainType::StaticFittedMesh &&
629  solverChoice.terrain_type != TerrainType::MovingFittedMesh) {
630  terrain_fab.template setVal<RunOn::Device>(0.0);
631  } else {
632  //
633  // Fill the values of the terrain height at k=0 only
634  //
635  prob->init_terrain_surface(geom[lev],terrain_fab,time);
636  }
637 
638  for (MFIter mfi(*z_phys_nd[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
639  {
640  Box isect = terrain_fab.box() & (*z_phys_nd[lev])[mfi].box();
641  if (!isect.isEmpty()) {
642  (*z_phys_nd[lev])[mfi].template copy<RunOn::Device>(terrain_fab,isect,0,isect,0,1);
643  }
644  }
645 
647 
648  z_phys_nd[lev]->FillBoundary(geom[lev].periodicity());
649 
650  if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
651  terrain_blanking[lev]->setVal(1.0);
652  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ngrow);
653  terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
654  }
655 
656  if (lev == 0) {
657  Real zmax = z_phys_nd[0]->max(0,0,false);
658  Real rel_diff = (zmax - zlevels_stag[0][zlevels_stag[0].size()-1]) / zmax;
659  if (rel_diff < 1.e-8) {
660  amrex::Print() << "max of zphys_nd " << zmax << std::endl;
661  amrex::Print() << "max of zlevels " << zlevels_stag[0][zlevels_stag[0].size()-1] << std::endl;
662  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rel_diff < 1.e-8, "Terrain is taller than domain top!");
663  }
664  } // lev == 0
665 
666  } // init_type
667 
668  // Compute the min dz and pass to the micro model
669  Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
670  micro->Set_dzmin(lev, dzmin);
671 }
Real get_dzmin_terrain(MultiFab &z_phys_nd)
Definition: ERF_TerrainMetrics.cpp:649
void make_terrain_fitted_coords(int lev, const Geometry &geom, MultiFab &z_phys_nd, Vector< Real > const &z_levels_h, GpuArray< ERF_BC, AMREX_SPACEDIM *2 > &phys_bc_type)
Definition: ERF_TerrainMetrics.cpp:46
Here is the call graph for this function:

◆ InitData()

void ERF::InitData ( )
891 {
892  BL_PROFILE_VAR("ERF::InitData()", InitData);
893  InitData_pre();
894  InitData_post();
895  BL_PROFILE_VAR_STOP(InitData);
896 }
void InitData_pre()
Definition: ERF.cpp:899
void InitData_post()
Definition: ERF.cpp:978
void InitData()
Definition: ERF.cpp:890

Referenced by main().

Here is the caller graph for this function:

◆ InitData_post()

void ERF::InitData_post ( )
979 {
981  {
982  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level == 0,
983  "Thin immersed body with refinement not currently supported.");
984  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
985  amrex::Print() << "NOTE: Thin immersed body with non-constant dz has not been tested." << std::endl;
986  }
987  }
988 
989  if (!restart_chkfile.empty()) {
990  restart();
991  }
992 
993  //
994  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
995  //
996  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
997  for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
998  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
999  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
1000  detJ_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
1001  z_phys_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
1002  }
1003  }
1004 
1005 #ifdef ERF_IMPLICIT_W
1006  if (SolverChoice::mesh_type == MeshType::VariableDz &&
1007  (solverChoice.vert_implicit_fac[0] > 0 ||
1009  solverChoice.vert_implicit_fac[2] > 0 ) &&
1011  {
1012  amrex::Warning("Doing implicit solve for u, v, and w with terrain -- this has not been tested");
1013  }
1014 #endif
1015 
1016  //
1017  // Copy vars_new into vars_old, then use vars_old to fill covered cells in vars_new during AverageDown
1018  //
1019  if (SolverChoice::terrain_type == TerrainType::EB) {
1020  for (int lev = 0; lev <= finest_level; lev++) {
1021  int ncomp_cons = vars_new[lev][Vars::cons].nComp();
1022  MultiFab::Copy(vars_old[lev][Vars::cons],vars_new[lev][Vars::cons],0,0,ncomp_cons,vars_new[lev][Vars::cons].nGrowVect());
1023  }
1024  }
1025 
1026  if (restart_chkfile.empty()) {
1027  if (solverChoice.coupling_type == CouplingType::TwoWay) {
1028  AverageDown();
1029  }
1030  }
1031 
1032 #ifdef ERF_USE_PARTICLES
1033  if (restart_chkfile.empty()) {
1034  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
1036  Warning("Tight coupling has not been tested with Lagrangian microphysics");
1037  }
1038 
1039  for (int lev = 0; lev <= finest_level; lev++) {
1040  dynamic_cast<LagrangianMicrophysics&>(*micro).initParticles(z_phys_nd[lev]);
1041  }
1042  }
1043  }
1044 #endif
1045 
1046  if (!restart_chkfile.empty()) { // Restart from a checkpoint
1047 
1048  // Create the physbc objects for {cons, u, v, w, base state}
1049  // We fill the additional base state ghost cells just in case we have read the old format
1050  for (int lev(0); lev <= finest_level; ++lev) {
1051  make_physbcs(lev);
1052  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
1053  }
1054 
1056  for (int lev(0); lev <= finest_level; ++lev) {
1057  m_forest_drag[lev]->define_drag_field(grids[lev], dmap[lev], geom[lev],
1058  z_phys_cc[lev].get(), z_phys_nd[lev].get());
1059  }
1060  }
1061 
1062 #ifdef ERF_USE_NETCDF
1063  //
1064  // Create the needed bdy_data_xlo etc ... since we don't read it in from checkpoint any more
1065  // This follows init_from_wrfinput()
1066  //
1067  bool use_moist = (solverChoice.moisture_type != MoistureType::None);
1068  if (solverChoice.use_real_bcs) {
1069 
1070  if ( geom[0].isPeriodic(0) || geom[0].isPeriodic(1) ) {
1071  amrex::Error("Cannot set periodic lateral boundary conditions when reading in real boundary values");
1072  }
1073 
1074  bdy_time_interval = read_times_from_wrfbdy(nc_bdy_file,
1075  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
1076  start_bdy_time);
1077  Real dT = bdy_time_interval;
1078 
1079  int n_time_old = static_cast<int>(t_new[0] / dT);
1080 
1081  int lev = 0;
1082 
1083  int ntimes = std::min(n_time_old+3, static_cast<int>(bdy_data_xlo.size()));
1084 
1085  for (int itime = n_time_old; itime < ntimes; itime++)
1086  {
1087  read_from_wrfbdy(itime,nc_bdy_file,geom[0].Domain(),
1088  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
1089  real_width);
1090  convert_all_wrfbdy_data(itime, geom[0].Domain(), bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi,
1091  *mf_MUB, *mf_C1H, *mf_C2H,
1093  geom[lev], use_moist);
1094  } // itime
1095  } // use_real_bcs
1096 
1097  if (!nc_low_file.empty())
1098  {
1099  low_time_interval = read_times_from_wrflow(nc_low_file,
1100  low_data_zlo,
1101  start_low_time);
1102  Real dT = low_time_interval;
1103 
1104  int lev = 0;
1105  sst_lev[lev].resize(low_data_zlo.size());
1106  tsk_lev[lev].resize(low_data_zlo.size());
1107 
1108  int n_time_old = static_cast<int>(t_new[0] / dT);
1109 
1110  int ntimes = std::min(n_time_old+2, static_cast<int>(low_data_zlo.size()));
1111 
1112  for (int itime = n_time_old; itime < ntimes; itime++)
1113  {
1114  read_from_wrflow(itime, nc_low_file, geom[lev].Domain(), low_data_zlo);
1115 
1116  // Need to read PSFC
1117  FArrayBox NC_fab_var_file;
1118  for (int idx = 0; idx < num_boxes_at_level[lev]; idx++) {
1119  int success, use_theta_m;
1120  read_from_wrfinput(lev, boxes_at_level[lev][idx], nc_init_file[lev][0],
1121  NC_fab_var_file, "PSFC", geom[lev],
1122  use_theta_m, success);
1123  auto& var_fab = NC_fab_var_file;
1124 #ifdef _OPENMP
1125 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1126 #endif
1127  for ( MFIter mfi(*mf_PSFC[lev], false); mfi.isValid(); ++mfi )
1128  {
1129  FArrayBox &cur_fab = (*mf_PSFC[lev])[mfi];
1130  cur_fab.template copy<RunOn::Device>(var_fab, 0, 0, 1);
1131  }
1132  var_fab.clear();
1133  }
1134 
1135  update_sst_tsk(itime, geom[lev], ba2d[lev],
1136  sst_lev[lev], tsk_lev[lev],
1137  m_SurfaceLayer, low_data_zlo,
1138  vars_new[lev][Vars::cons], *mf_PSFC[lev],
1139  solverChoice.rdOcp, lmask_lev[lev][0], use_moist);
1140  } // itime
1141  }
1142 #endif
1143  } // end restart
1144 
1145 #ifdef ERF_USE_PARTICLES
1146  /* If using a Lagrangian microphysics model, its particle container has now been
1147  constructed and initialized (calls to micro->Init). So, add its pointer to
1148  ERF::particleData and remove its name from list of unallocated particle containers. */
1149  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
1150  const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
1151  const auto& pc_ptr( dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer() );
1152  particleData.pushBack(pc_name, pc_ptr);
1153  particleData.getNamesUnalloc().remove(pc_name);
1154  }
1155 #endif
1156 
1157  if (input_bndry_planes) {
1158  // Read the "time.dat" file to know what data is available
1159  m_r2d->read_time_file();
1160 
1161  // We haven't populated dt yet, set to 0 to ensure assert doesn't crash
1162  Real dt_dummy = 0.0;
1163  m_r2d->read_input_files(t_new[0],dt_dummy,m_bc_extdir_vals);
1164  }
1165 
1167  {
1168  h_rhotheta_src.resize(max_level+1, Vector<Real>(0));
1169  d_rhotheta_src.resize(max_level+1, Gpu::DeviceVector<Real>(0));
1170  for (int lev = 0; lev <= finest_level; lev++) {
1171  const int domlen = geom[lev].Domain().length(2);
1172  h_rhotheta_src[lev].resize(domlen, 0.0_rt);
1173  d_rhotheta_src[lev].resize(domlen, 0.0_rt);
1174  prob->update_rhotheta_sources(t_new[0],
1175  h_rhotheta_src[lev], d_rhotheta_src[lev],
1176  geom[lev], z_phys_cc[lev]);
1177  }
1178  }
1179 
1181  {
1182  h_u_geos.resize(max_level+1, Vector<Real>(0));
1183  d_u_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
1184  h_v_geos.resize(max_level+1, Vector<Real>(0));
1185  d_v_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
1186  for (int lev = 0; lev <= finest_level; lev++) {
1187  const int domlen = geom[lev].Domain().length(2);
1188  h_u_geos[lev].resize(domlen, 0.0_rt);
1189  d_u_geos[lev].resize(domlen, 0.0_rt);
1190  h_v_geos[lev].resize(domlen, 0.0_rt);
1191  d_v_geos[lev].resize(domlen, 0.0_rt);
1193  prob->update_geostrophic_profile(t_new[0],
1194  h_u_geos[lev], d_u_geos[lev],
1195  h_v_geos[lev], d_v_geos[lev],
1196  geom[lev], z_phys_cc[lev]);
1197  } else {
1198  if (SolverChoice::mesh_type == MeshType::VariableDz) {
1199  amrex::Print() << "Note: 1-D geostrophic wind profile input is not defined for real terrain" << std::endl;
1200  }
1202  h_u_geos[lev], d_u_geos[lev],
1203  h_v_geos[lev], d_v_geos[lev],
1204  geom[lev],
1205  zlevels_stag[0]);
1206  }
1207  }
1208  }
1209 
1211  {
1212  h_rhoqt_src.resize(max_level+1, Vector<Real>(0));
1213  d_rhoqt_src.resize(max_level+1, Gpu::DeviceVector<Real>(0));
1214  for (int lev = 0; lev <= finest_level; lev++) {
1215  const int domlen = geom[lev].Domain().length(2);
1216  h_rhoqt_src[lev].resize(domlen, 0.0_rt);
1217  d_rhoqt_src[lev].resize(domlen, 0.0_rt);
1218  prob->update_rhoqt_sources(t_new[0],
1219  h_rhoqt_src[lev], d_rhoqt_src[lev],
1220  geom[lev], z_phys_cc[lev]);
1221  }
1222  }
1223 
1225  {
1226  h_w_subsid.resize(max_level+1, Vector<Real>(0));
1227  d_w_subsid.resize(max_level+1, Gpu::DeviceVector<Real>(0));
1228  for (int lev = 0; lev <= finest_level; lev++) {
1229  const int domlen = geom[lev].Domain().length(2) + 1; // lives on z-faces
1230  h_w_subsid[lev].resize(domlen, 0.0_rt);
1231  d_w_subsid[lev].resize(domlen, 0.0_rt);
1232  prob->update_w_subsidence(t_new[0],
1233  h_w_subsid[lev], d_w_subsid[lev],
1234  geom[lev], z_phys_nd[lev]);
1235  }
1236  }
1237 
1240  {
1241  initRayleigh();
1242  if (solverChoice.init_type == InitType::Input_Sounding)
1243  {
1244  // Overwrite ubar, vbar, and thetabar with input profiles;
1245  // wbar is assumed to be 0. Note: the tau coefficient set by
1246  // prob->erf_init_rayleigh() is still used
1247  bool restarting = (!restart_chkfile.empty());
1248  setRayleighRefFromSounding(restarting);
1249  }
1250  }
1251 
1252  // Read in sponge data from input file
1253  if(solverChoice.spongeChoice.sponge_type == "input_sponge")
1254  {
1255  initSponge();
1256  bool restarting = (!restart_chkfile.empty());
1257  setSpongeRefFromSounding(restarting);
1258  }
1259 
1260  if (solverChoice.pert_type == PerturbationType::Source ||
1261  solverChoice.pert_type == PerturbationType::Direct ||
1262  solverChoice.pert_type == PerturbationType::CPM) {
1263  if (is_it_time_for_action(istep[0], t_new[0], dt[0], pert_interval, -1.)) {
1264  turbPert.debug(t_new[0]);
1265  }
1266  }
1267 
1268  // We only write the file at level 0 for now
1269  if (output_bndry_planes)
1270  {
1271  // Create the WriteBndryPlanes object so we can handle writing of boundary plane data
1272  m_w2d = std::make_unique<WriteBndryPlanes>(grids,geom);
1273 
1274  Real time = 0.;
1275  if (time >= bndry_output_planes_start_time) {
1276  bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
1277  m_w2d->write_planes(0, time, vars_new, is_moist);
1278  }
1279  }
1280 
1281  // Fill boundary conditions in vars_new
1282  for (int lev = 0; lev <= finest_level; ++lev)
1283  {
1284  auto& lev_new = vars_new[lev];
1285 
1286  // ***************************************************************************
1287  // Physical bc's at domain boundary
1288  // ***************************************************************************
1289  IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
1290  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
1291 
1292  int ncomp_cons = lev_new[Vars::cons].nComp();
1293  bool do_fb = true;
1294 
1295 #ifdef ERF_USE_NETCDF
1296  // We call this here because it is an ERF routine
1297  if (solverChoice.use_real_bcs && (lev==0)) {
1298  int icomp_cons = 0;
1299  bool cons_only = false;
1300  Vector<MultiFab*> mfs_vec = {&lev_new[Vars::cons],&lev_new[Vars::xvel],
1301  &lev_new[Vars::yvel],&lev_new[Vars::zvel]};
1303  fill_from_realbdy_upwind(mfs_vec,t_new[lev],cons_only,icomp_cons,
1304  ncomp_cons,ngvect_cons,ngvect_vels);
1305  } else {
1306  fill_from_realbdy(mfs_vec,t_new[lev],cons_only,icomp_cons,
1307  ncomp_cons,ngvect_cons,ngvect_vels);
1308  }
1309  do_fb = false;
1310  }
1311 #endif
1312 
1313  (*physbcs_cons[lev])(lev_new[Vars::cons],lev_new[Vars::xvel],lev_new[Vars::yvel],0,ncomp_cons,
1314  ngvect_cons,t_new[lev],BCVars::cons_bc,do_fb);
1315  ( *physbcs_u[lev])(lev_new[Vars::xvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
1316  ngvect_vels,t_new[lev],BCVars::xvel_bc,do_fb);
1317  ( *physbcs_v[lev])(lev_new[Vars::yvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
1318  ngvect_vels,t_new[lev],BCVars::yvel_bc,do_fb);
1319  ( *physbcs_w[lev])(lev_new[Vars::zvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
1320  ngvect_vels,t_new[lev],BCVars::zvel_bc,do_fb);
1321  }
1322 
1323  //
1324  // If we are starting from scratch, we have the option to project the initial velocity field
1325  // regardless of how we initialized. Note that project_velocity operates on vars_new.
1326  // pp_inc is used as scratch space here; we zero it out after the projection
1327  //
1328  if (restart_chkfile == "")
1329  {
1331  Real dummy_dt = 1.0;
1332  if (verbose > 0) {
1333  amrex::Print() << "Projecting initial velocity field" << std::endl;
1334  }
1335  for (int lev = 0; lev <= finest_level; ++lev)
1336  {
1337  project_velocity(lev, dummy_dt);
1338  pp_inc[lev].setVal(0.);
1339  gradp[lev][GpVars::gpx].setVal(0.);
1340  gradp[lev][GpVars::gpy].setVal(0.);
1341  gradp[lev][GpVars::gpz].setVal(0.);
1342  }
1343  }
1344  }
1345 
1346  // Copy from new into old just in case (after filling boundary conditions and possibly projecting)
1347  for (int lev = 0; lev <= finest_level; ++lev)
1348  {
1349  int nc = vars_new[lev][Vars::cons].nComp();
1350 
1351  MultiFab::Copy(vars_old[lev][Vars::cons],vars_new[lev][Vars::cons],0,0,nc,vars_new[lev][Vars::cons].nGrowVect());
1352  MultiFab::Copy(vars_old[lev][Vars::xvel],vars_new[lev][Vars::xvel],0,0, 1,vars_new[lev][Vars::xvel].nGrowVect());
1353  MultiFab::Copy(vars_old[lev][Vars::yvel],vars_new[lev][Vars::yvel],0,0, 1,vars_new[lev][Vars::yvel].nGrowVect());
1354  MultiFab::Copy(vars_old[lev][Vars::zvel],vars_new[lev][Vars::zvel],0,0, 1,vars_new[lev][Vars::zvel].nGrowVect());
1355  }
1356 
1357  // Compute the minimum dz in the domain at each level (to be used for setting the timestep)
1358  dz_min.resize(max_level+1);
1359  for (int lev = 0; lev <= finest_level; ++lev)
1360  {
1361  dz_min[lev] = geom[lev].CellSize(2);
1362  if ( SolverChoice::mesh_type != MeshType::ConstantDz ) {
1363  dz_min[lev] *= (*detJ_cc[lev]).min(0);
1364  }
1365  }
1366 
1367  // We don't need to recompute dt[lev] on restart because we read it in from the checkpoint file.
1368  if (restart_chkfile.empty()) {
1369  ComputeDt();
1370  }
1371 
1372  // Check the viscous limit
1376  Real delta = std::min({geom[finest_level].CellSize(0),
1377  geom[finest_level].CellSize(1),
1378  dz_min[finest_level]});
1379  if (dc.dynamic_viscosity == 0) {
1380  Print() << "Note: Molecular diffusion specified but dynamic_viscosity has not been specified" << std::endl;
1381  } else {
1382  Real nu = dc.dynamic_viscosity / dc.rho0_trans;
1383  Real viscous_limit = 2.0 * delta*delta / nu;
1384  Print() << "smallest grid spacing at level " << finest_level << " = " << delta << std::endl;
1385  Print() << "dt at level " << finest_level << " = " << dt[finest_level] << std::endl;
1386  Print() << "Viscous CFL is " << dt[finest_level] / viscous_limit << std::endl;
1387  if (fixed_dt[finest_level] >= viscous_limit) {
1388  Warning("Specified fixed_dt is above the viscous limit");
1389  } else if (dt[finest_level] >= viscous_limit) {
1390  Warning("Adaptive dt based on convective CFL only is above the viscous limit");
1391  }
1392  }
1393  }
1394 
1395  // Fill ghost cells/faces
1396  for (int lev = 0; lev <= finest_level; ++lev)
1397  {
1398  if (lev > 0 && cf_width >= 0) {
1400  }
1401 
1402  auto& lev_new = vars_new[lev];
1403 
1404  //
1405  // Fill boundary conditions -- not sure why we need this here
1406  //
1407  bool fillset = false;
1408  if (lev == 0) {
1409  FillPatchCrseLevel(lev, t_new[lev],
1410  {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]});
1411  } else {
1412  FillPatchFineLevel(lev, t_new[lev],
1413  {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]},
1414  {&lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
1415  base_state[lev], base_state[lev],
1416  fillset);
1417  }
1418 
1419  //
1420  // We do this here to make sure level (lev-1) boundary conditions are filled
1421  // before we interpolate to level (lev) ghost cells
1422  //
1423  if (lev < finest_level) {
1424  auto& lev_old = vars_old[lev];
1425  MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,lev_old[Vars::cons].nComp(),lev_old[Vars::cons].nGrowVect());
1426  MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0,lev_old[Vars::xvel].nComp(),lev_old[Vars::xvel].nGrowVect());
1427  MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0,lev_old[Vars::yvel].nComp(),lev_old[Vars::yvel].nGrowVect());
1428  MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0,lev_old[Vars::zvel].nComp(),lev_old[Vars::zvel].nGrowVect());
1429  }
1430 
1431  //
1432  // We fill the ghost cell values of the base state in case it wasn't done in the initialization
1433  //
1434  base_state[lev].FillBoundary(geom[lev].periodicity());
1435 
1436  // For moving terrain only
1437  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
1438  MultiFab::Copy(base_state_new[lev],base_state[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
1439  base_state_new[lev].FillBoundary(geom[lev].periodicity());
1440  }
1441 
1442  }
1443 
1444  // Allow idealized cases over water, used to set lmask
1445  ParmParse pp("erf");
1446  int is_land;
1447  for (int lev = 0; lev <= finest_level; ++lev)
1448  {
1449  if (pp.query("is_land", is_land, lev)) {
1450  if (is_land == 1) {
1451  amrex::Print() << "Level " << lev << " is land" << std::endl;
1452  } else if (is_land == 0) {
1453  amrex::Print() << "Level " << lev << " is water" << std::endl;
1454  } else {
1455  Error("is_land should be 0 or 1");
1456  }
1457  lmask_lev[lev][0]->setVal(is_land);
1458  lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
1459  }
1460  }
1461 
1462  // If lev > 0, we need to fill bc's by interpolation from coarser grid
1463  for (int lev = 1; lev <= finest_level; ++lev)
1464  {
1465  Real time_for_fp = 0.; // This is not actually used
1466  Vector<Real> ftime = {time_for_fp, time_for_fp};
1467  Vector<Real> ctime = {time_for_fp, time_for_fp};
1468  if (lat_m[lev]) {
1469  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1470  Vector<MultiFab*> fmf = {lat_m[lev ].get(), lat_m[lev ].get()};
1471  Vector<MultiFab*> cmf = {lat_m[lev-1].get(), lat_m[lev-1].get()};
1472  IntVect ngv = lat_m[lev]->nGrowVect(); ngv[2] = 0;
1473  Interpolater* mapper = &cell_cons_interp;
1474  FillPatchTwoLevels(*lat_m[lev].get(), ngv, IntVect(0,0,0),
1475  time_for_fp, cmf, ctime, fmf, ftime,
1476  0, 0, 1, geom[lev-1], geom[lev],
1477  refRatio(lev-1), mapper, domain_bcs_type,
1478  BCVars::cons_bc);
1479  }
1480  if (lon_m[lev]) {
1481  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1482  Vector<MultiFab*> fmf = {lon_m[lev ].get(), lon_m[lev ].get()};
1483  Vector<MultiFab*> cmf = {lon_m[lev-1].get(), lon_m[lev-1].get()};
1484  IntVect ngv = lon_m[lev]->nGrowVect(); ngv[2] = 0;
1485  Interpolater* mapper = &cell_cons_interp;
1486  FillPatchTwoLevels(*lon_m[lev].get(), ngv, IntVect(0,0,0),
1487  time_for_fp, cmf, ctime, fmf, ftime,
1488  0, 0, 1, geom[lev-1], geom[lev],
1489  refRatio(lev-1), mapper, domain_bcs_type,
1490  BCVars::cons_bc);
1491  } // lon_m
1492  if (sinPhi_m[lev]) {
1493  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1494  Vector<MultiFab*> fmf = {sinPhi_m[lev ].get(), sinPhi_m[lev ].get()};
1495  Vector<MultiFab*> cmf = {sinPhi_m[lev-1].get(), sinPhi_m[lev-1].get()};
1496  IntVect ngv = sinPhi_m[lev]->nGrowVect(); ngv[2] = 0;
1497  Interpolater* mapper = &cell_cons_interp;
1498  FillPatchTwoLevels(*sinPhi_m[lev].get(), ngv, IntVect(0,0,0),
1499  time_for_fp, cmf, ctime, fmf, ftime,
1500  0, 0, 1, geom[lev-1], geom[lev],
1501  refRatio(lev-1), mapper, domain_bcs_type,
1502  BCVars::cons_bc);
1503  } // sinPhi
1504  if (cosPhi_m[lev]) {
1505  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1506  Vector<MultiFab*> fmf = {cosPhi_m[lev ].get(), cosPhi_m[lev ].get()};
1507  Vector<MultiFab*> cmf = {cosPhi_m[lev-1].get(), cosPhi_m[lev-1].get()};
1508  IntVect ngv = cosPhi_m[lev]->nGrowVect(); ngv[2] = 0;
1509  Interpolater* mapper = &cell_cons_interp;
1510  FillPatchTwoLevels(*cosPhi_m[lev].get(), ngv, IntVect(0,0,0),
1511  time_for_fp, cmf, ctime, fmf, ftime,
1512  0, 0, 1, geom[lev-1], geom[lev],
1513  refRatio(lev-1), mapper, domain_bcs_type,
1514  BCVars::cons_bc);
1515  } // cosPhi
1516  } // lev
1517 
1518 #ifdef ERF_USE_WW3_COUPLING
1519  int my_lev = 0;
1520  amrex::Print() << " About to call send_to_ww3 from ERF.cpp" << std::endl;
1521  send_to_ww3(my_lev);
1522  amrex::Print() << " About to call read_waves from ERF.cpp" << std::endl;
1523  read_waves(my_lev);
1524  // send_to_ww3(my_lev);
1525 #endif
1526 
1527  // Configure SurfaceLayer params if used
1528  // NOTE: we must set up the MOST routine after calling FillPatch
1529  // in order to have lateral ghost cells filled (MOST + terrain interp).
1530  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
1531  {
1533  (solverChoice.turbChoice[0].les_type != LESType::None) ||
1534  (solverChoice.turbChoice[0].pbl_type != PBLType::None) );
1535  AMREX_ALWAYS_ASSERT(has_diff);
1536 
1537  bool rotate = solverChoice.use_rotate_surface_flux;
1538  if (rotate) {
1539  Print() << "Using surface layer model with stress rotations" << std::endl;
1540  }
1541 
1542  //
1543  // This constructor will make the SurfaceLayer object but not allocate the arrays at each level.
1544  //
1545  m_SurfaceLayer = std::make_unique<SurfaceLayer>(geom, rotate, pp_prefix, Qv_prim,
1546  z_phys_nd,
1550 #ifdef ERF_USE_NETCDF
1551  , bdy_time_interval
1552 #endif
1553  );
1554  // This call will allocate the arrays at each level. If we regrid later, either changing
1555  // the number of levels or just the grids at each existing level, we will call an update routine
1556  // to redefine the internal arrays in m_SurfaceLayer.
1557  for (int lev = 0; lev <= finest_level; lev++)
1558  {
1559  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
1560  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
1561  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,finest_level+1,
1562  mfv_old, Theta_prim[lev], Qv_prim[lev],
1563  Qr_prim[lev], z_phys_nd[lev],
1564  Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
1566  sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
1567  }
1568 
1569 
1570  if (restart_chkfile != "") {
1571  // Update surface fields if needed (and available)
1573  }
1574 
1575  // We now configure ABLMost params here so that we can print the averages at t=0
1576  // Note we don't fill ghost cells here because this is just for diagnostics
1577  for (int lev = 0; lev <= finest_level; ++lev)
1578  {
1579  Real time = t_new[lev];
1580  IntVect ng = Theta_prim[lev]->nGrowVect();
1581 
1582  MultiFab::Copy( *Theta_prim[lev], vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, ng);
1583  MultiFab::Divide(*Theta_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1584 
1585  if (solverChoice.moisture_type != MoistureType::None) {
1586  ng = Qv_prim[lev]->nGrowVect();
1587 
1588  MultiFab::Copy( *Qv_prim[lev], vars_new[lev][Vars::cons], RhoQ1_comp, 0, 1, ng);
1589  MultiFab::Divide(*Qv_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1590 
1591  int rhoqr_comp = solverChoice.moisture_indices.qr;
1592  if (rhoqr_comp > -1) {
1593  MultiFab::Copy( *Qr_prim[lev], vars_new[lev][Vars::cons], rhoqr_comp, 0, 1, ng);
1594  MultiFab::Divide(*Qr_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1595  } else {
1596  Qr_prim[lev]->setVal(0.0);
1597  }
1598  }
1599  m_SurfaceLayer->update_mac_ptrs(lev, vars_new, Theta_prim, Qv_prim, Qr_prim);
1600 
1601  if (restart_chkfile == "") {
1602  // Only do this if starting from scratch; if restarting, then
1603  // we don't want to call update_fluxes multiple times because
1604  // it will change u* and theta* from their previous values
1605  m_SurfaceLayer->update_pblh(lev, vars_new, z_phys_cc[lev].get(),
1607  m_SurfaceLayer->update_fluxes(lev, time, vars_new[lev][Vars::cons], z_phys_nd[lev]);
1608 
1609  // Initialize tke(x,y,z) as a function of u*(x,y)
1610  if (solverChoice.turbChoice[lev].init_tke_from_ustar) {
1611  Real qkefac = 1.0;
1612  if (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25 ||
1613  solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF)
1614  {
1615  // https://github.com/NCAR/MYNN-EDMF/blob/90f36c25259ec1960b24325f5b29ac7c5adeac73/module_bl_mynnedmf.F90#L1325-L1333
1616  const Real B1 = solverChoice.turbChoice[lev].pbl_mynn.B1;
1617  qkefac = 1.5 * std::pow(B1, 2.0/3.0);
1618  }
1619  m_SurfaceLayer->init_tke_from_ustar(lev, vars_new[lev][Vars::cons], z_phys_nd[lev], qkefac);
1620  }
1621  }
1622  }
1623  } // end if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
1624 
1625  // Update micro vars before first plot file
1626  if (solverChoice.moisture_type != MoistureType::None) {
1627  for (int lev = 0; lev <= finest_level; ++lev) micro->Update_Micro_Vars_Lev(lev, vars_new[lev][Vars::cons]);
1628  }
1629 
1630  // Fill time averaged velocities before first plot file
1631  if (solverChoice.time_avg_vel) {
1632  for (int lev = 0; lev <= finest_level; ++lev) {
1633  Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(),
1634  vars_new[lev][Vars::xvel],
1635  vars_new[lev][Vars::yvel],
1636  vars_new[lev][Vars::zvel]);
1637  }
1638  }
1639 
1640  // check for additional plotting variables that are available after particle containers
1641  // are setup.
1642  const std::string& pv3d_1 = "plot_vars_1" ; appendPlotVariables(pv3d_1,plot3d_var_names_1);
1643  const std::string& pv3d_2 = "plot_vars_2" ; appendPlotVariables(pv3d_2,plot3d_var_names_2);
1644  const std::string& pv2d_1 = "plot2d_vars_1"; appendPlotVariables(pv2d_1,plot2d_var_names_1);
1645  const std::string& pv2d_2 = "plot2d_vars_2"; appendPlotVariables(pv2d_2,plot2d_var_names_2);
1646 
1647  if ( restart_chkfile.empty() && (m_check_int > 0 || m_check_per > 0.) )
1648  {
1652  }
1653 
1654  if ( (restart_chkfile.empty()) ||
1655  (!restart_chkfile.empty() && plot_file_on_restart) )
1656  {
1657  if (m_plot3d_int_1 > 0 || m_plot3d_per_1 > 0.)
1658  {
1662  }
1663  if (m_plot3d_int_2 > 0 || m_plot3d_per_2 > 0.)
1664  {
1668  }
1669  if (m_plot2d_int_1 > 0 || m_plot2d_per_1 > 0.)
1670  {
1674  }
1675  if (m_plot2d_int_2 > 0 || m_plot2d_per_2 > 0.)
1676  {
1680  }
1681  if (m_subvol_int > 0 || m_subvol_per > 0.) {
1683  last_subvol_step = istep[0];
1685  }
1686  }
1687 
1688  // Set these up here because we need to know which MPI rank "cell" is on...
1689  if (pp.contains("data_log"))
1690  {
1691  int num_datalogs = pp.countval("data_log");
1692  datalog.resize(num_datalogs);
1693  datalogname.resize(num_datalogs);
1694  pp.queryarr("data_log",datalogname,0,num_datalogs);
1695  for (int i = 0; i < num_datalogs; i++) {
1697  }
1698  }
1699 
1700  if (pp.contains("der_data_log"))
1701  {
1702  int num_der_datalogs = pp.countval("der_data_log");
1703  der_datalog.resize(num_der_datalogs);
1704  der_datalogname.resize(num_der_datalogs);
1705  pp.queryarr("der_data_log",der_datalogname,0,num_der_datalogs);
1706  for (int i = 0; i < num_der_datalogs; i++) {
1708  }
1709  }
1710 
1711  if (pp.contains("energy_data_log"))
1712  {
1713  int num_energy_datalogs = pp.countval("energy_data_log");
1714  tot_e_datalog.resize(num_energy_datalogs);
1715  tot_e_datalogname.resize(num_energy_datalogs);
1716  pp.queryarr("energy_data_log",tot_e_datalogname,0,num_energy_datalogs);
1717  for (int i = 0; i < num_energy_datalogs; i++) {
1719  }
1720  }
1721 
1722  if (solverChoice.rad_type != RadiationType::None)
1723  {
1724  // Create data log for radiation model if requested
1725  rad[0]->setupDataLog();
1726  }
1727 
1728 
1729  if (restart_chkfile.empty() && profile_int > 0) {
1730  if (destag_profiles) {
1731  // all variables cell-centered
1733  } else {
1734  // some variables staggered
1736  }
1737  }
1738 
1739  if (pp.contains("sample_point_log") && pp.contains("sample_point"))
1740  {
1741  int lev = 0;
1742 
1743  int num_samplepts = pp.countval("sample_point") / AMREX_SPACEDIM;
1744  if (num_samplepts > 0) {
1745  Vector<int> index; index.resize(num_samplepts*AMREX_SPACEDIM);
1746  samplepoint.resize(num_samplepts);
1747 
1748  pp.queryarr("sample_point",index,0,num_samplepts*AMREX_SPACEDIM);
1749  for (int i = 0; i < num_samplepts; i++) {
1750  IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
1751  samplepoint[i] = iv;
1752  }
1753  }
1754 
1755  int num_sampleptlogs = pp.countval("sample_point_log");
1756  AMREX_ALWAYS_ASSERT(num_sampleptlogs == num_samplepts);
1757  if (num_sampleptlogs > 0) {
1758  sampleptlog.resize(num_sampleptlogs);
1759  sampleptlogname.resize(num_sampleptlogs);
1760  pp.queryarr("sample_point_log",sampleptlogname,0,num_sampleptlogs);
1761 
1762  for (int i = 0; i < num_sampleptlogs; i++) {
1764  }
1765  }
1766 
1767  }
1768 
1769  if (pp.contains("sample_line_log") && pp.contains("sample_line"))
1770  {
1771  int lev = 0;
1772 
1773  int num_samplelines = pp.countval("sample_line") / AMREX_SPACEDIM;
1774  if (num_samplelines > 0) {
1775  Vector<int> index; index.resize(num_samplelines*AMREX_SPACEDIM);
1776  sampleline.resize(num_samplelines);
1777 
1778  pp.queryarr("sample_line",index,0,num_samplelines*AMREX_SPACEDIM);
1779  for (int i = 0; i < num_samplelines; i++) {
1780  IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
1781  sampleline[i] = iv;
1782  }
1783  }
1784 
1785  int num_samplelinelogs = pp.countval("sample_line_log");
1786  AMREX_ALWAYS_ASSERT(num_samplelinelogs == num_samplelines);
1787  if (num_samplelinelogs > 0) {
1788  samplelinelog.resize(num_samplelinelogs);
1789  samplelinelogname.resize(num_samplelinelogs);
1790  pp.queryarr("sample_line_log",samplelinelogname,0,num_samplelinelogs);
1791 
1792  for (int i = 0; i < num_samplelinelogs; i++) {
1794  }
1795  }
1796 
1797  }
1798 
1803  }
1804 
1805  // Create object to do line and plane sampling if needed
1806  bool do_line = false; bool do_plane = false;
1807  pp.query("do_line_sampling",do_line); pp.query("do_plane_sampling",do_plane);
1808  if (do_line) {
1809  if (line_sampling_interval < 0 && line_sampling_per < 0) {
1810  Abort("Need to specify line_sampling_interval or line_sampling_per");
1811  }
1812  line_sampler = std::make_unique<LineSampler>();
1813  line_sampler->write_coords(z_phys_cc);
1814  }
1815  if (do_plane) {
1817  Abort("Need to specify plane_sampling_interval or plane_sampling_per");
1818  }
1819  plane_sampler = std::make_unique<PlaneSampler>();
1820  }
1821 
1822  if ( solverChoice.terrain_type == TerrainType::EB ||
1823  solverChoice.terrain_type == TerrainType::ImmersedForcing)
1824  {
1825  bool write_eb_surface = false;
1826  pp.query("write_eb_surface", write_eb_surface);
1827  if (write_eb_surface) WriteMyEBSurface();
1828  }
1829 
1830 }
void initRayleigh()
Initialize Rayleigh damping profiles.
Definition: ERF_InitRayleigh.cpp:14
amrex::Vector< std::string > samplelinelogname
Definition: ERF.H:1606
void setRayleighRefFromSounding(bool restarting)
Set Rayleigh mean profiles from input sounding.
Definition: ERF_InitRayleigh.cpp:94
amrex::Vector< amrex::IntVect > sampleline
Definition: ERF.H:1607
amrex::Real plane_sampling_per
Definition: ERF.H:1590
static amrex::Real sum_per
Definition: ERF.H:1204
void setRecordDataInfo(int i, const std::string &filename)
Definition: ERF.H:1513
static bool plot_file_on_restart
Definition: ERF.H:1018
amrex::Vector< std::string > lsm_flux_name
Definition: ERF.H:875
void WriteMyEBSurface()
Definition: ERF_EBWriteSurface.cpp:5
void write_1D_profiles_stag(amrex::Real time)
Definition: ERF_Write1DProfiles_stag.cpp:25
void sum_energy_quantities(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:318
amrex::Vector< std::unique_ptr< std::fstream > > samplelinelog
Definition: ERF.H:1605
static int sum_interval
Definition: ERF.H:1202
static int pert_interval
Definition: ERF.H:1203
amrex::Real line_sampling_per
Definition: ERF.H:1589
void restart()
Definition: ERF.cpp:1869
void write_1D_profiles(amrex::Real time)
Definition: ERF_Write1DProfiles.cpp:17
int profile_int
Definition: ERF.H:1085
bool destag_profiles
Definition: ERF.H:1086
void appendPlotVariables(const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
Definition: ERF_Plotfile.cpp:228
amrex::Vector< std::string > tot_e_datalogname
Definition: ERF.H:1599
static int output_bndry_planes
Definition: ERF.H:1261
static std::string nc_bdy_file
Definition: ERF.H:1221
void AverageDown()
Definition: ERF_AverageDown.cpp:16
static amrex::Real bndry_output_planes_start_time
Definition: ERF.H:1264
void project_velocity(int lev, amrex::Real dt)
Definition: ERF_PoissonSolve.cpp:10
std::string restart_chkfile
Definition: ERF.H:1040
amrex::Vector< std::string > sampleptlogname
Definition: ERF.H:1602
void sum_derived_quantities(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:187
void sum_integrated_quantities(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:15
void setRecordDerDataInfo(int i, const std::string &filename)
Definition: ERF.H:1526
amrex::Vector< std::unique_ptr< std::fstream > > sampleptlog
Definition: ERF.H:1601
std::unique_ptr< WriteBndryPlanes > m_w2d
Definition: ERF.H:1327
void init_geo_wind_profile(const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)
Definition: ERF_InitGeowind.cpp:10
amrex::Vector< std::string > lsm_data_name
Definition: ERF.H:873
void initSponge()
Initialize sponge profiles.
Definition: ERF_InitSponge.cpp:35
std::unique_ptr< PlaneSampler > plane_sampler
Definition: ERF.H:1592
amrex::Vector< std::unique_ptr< std::fstream > > tot_e_datalog
Definition: ERF.H:1596
int real_width
Definition: ERF.H:1222
void setRecordEnergyDataInfo(int i, const std::string &filename)
Definition: ERF.H:1539
int plane_sampling_interval
Definition: ERF.H:1588
static bool is_it_time_for_action(int nstep, amrex::Real time, amrex::Real dt, int action_interval, amrex::Real action_per)
Definition: ERF_WriteScalarProfiles.cpp:653
static std::string nc_low_file
Definition: ERF.H:1227
void Construct_ERFFillPatchers(int lev)
Definition: ERF.cpp:2689
void setRecordSampleLineInfo(int i, int lev, amrex::IntVect &cell, const std::string &filename)
Definition: ERF.H:1569
void setSpongeRefFromSounding(bool restarting)
Set sponge mean profiles from input sounding.
Definition: ERF_InitSponge.cpp:65
int line_sampling_interval
Definition: ERF.H:1587
amrex::Vector< amrex::IntVect > samplepoint
Definition: ERF.H:1603
std::unique_ptr< LineSampler > line_sampler
Definition: ERF.H:1591
void setRecordSamplePointInfo(int i, int lev, amrex::IntVect &cell, const std::string &filename)
Definition: ERF.H:1552
void ReadCheckpointFileSurfaceLayer()
Definition: ERF_Checkpoint.cpp:1039
static MoistureModelType modelType(const MoistureType a_moisture_type)
query if a specified moisture model is Eulerian or Lagrangian
Definition: ERF_Microphysics.H:90
@ nc
Definition: ERF_Morrison.H:44
bool have_zero_flux_faces
Definition: ERF_AdvStruct.H:440
amrex::Real rho0_trans
Definition: ERF_DiffStruct.H:91
amrex::Real dynamic_viscosity
Definition: ERF_DiffStruct.H:96
bool have_geo_wind_profile
Definition: ERF_DataStruct.H:1037
amrex::Vector< amrex::Real > vert_implicit_fac
Definition: ERF_DataStruct.H:937
std::string abl_geo_wind_table
Definition: ERF_DataStruct.H:1036
bool implicit_momentum_diffusion
Definition: ERF_DataStruct.H:940
bool use_rotate_surface_flux
Definition: ERF_DataStruct.H:1008
bool do_forest_drag
Definition: ERF_DataStruct.H:1058
void debug(amrex::Real)
Definition: ERF_TurbPertStruct.H:613
Here is the call graph for this function:

◆ InitData_pre()

void ERF::InitData_pre ( )
900 {
901  // Initialize the start time for our CPU-time tracker
902  startCPUTime = ParallelDescriptor::second();
903 
904  // Create the ReadBndryPlanes object so we can read boundary plane data
905  // m_r2d is used by init_bcs so we must instantiate this class before
906  if (input_bndry_planes) {
907  Print() << "Defining r2d for the first time " << std::endl;
908  m_r2d = std::make_unique<ReadBndryPlanes>(geom[0], solverChoice.rdOcp);
909  }
910 
911  if (restart_chkfile.empty()) {
912  // Start simulation from the beginning
913  InitFromScratch(0.0);
914  } else {
915  // For initialization this is done in init_only; it is done here for restart
916  init_bcs();
917  }
918 
919  // Verify solver choices
920  for (int lev(0); lev <= max_level; ++lev) {
921  // BC compatibility
922  if ( ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
923  (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF) ||
924  (solverChoice.turbChoice[lev].pbl_type == PBLType::YSU) ||
925  (solverChoice.turbChoice[lev].pbl_type == PBLType::MRF)
926  ) &&
927  phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::surface_layer ) {
928  Abort("MYNN2.5/MYNNEDMF/YSU/MRF PBL Model requires MOST at lower boundary");
929  }
930  if ( (solverChoice.turbChoice[lev].les_type == LESType::Deardorff) &&
931  (solverChoice.turbChoice[lev].Ce_wall > 0) &&
932  (phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::surface_layer) &&
933  (phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::slip_wall) &&
934  (phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::no_slip_wall) )
935  {
936  Warning("Deardorff LES assumes wall at zlo when applying Ce_wall");
937  }
938 
939  if ( (solverChoice.const_massflux_u != 0) &&
940  (phys_bc_type[Orientation(Direction::x,Orientation::low)] != ERF_BC::periodic ) )
941  {
942  Abort("Constant mass flux (in x) should be used with periodic boundaries");
943  }
944  if ( (solverChoice.const_massflux_v != 0) &&
945  (phys_bc_type[Orientation(Direction::y,Orientation::low)] != ERF_BC::periodic ) )
946  {
947  Abort("Constant mass flux (in y) should be used with periodic boundaries");
948  }
949 
950  // mesoscale diffusion
951  if ((geom[lev].CellSize(0) > 2000.) || (geom[lev].CellSize(1) > 2000.))
952  {
953  if ( (solverChoice.turbChoice[lev].les_type == LESType::Smagorinsky) &&
954  (!solverChoice.turbChoice[lev].smag2d)) {
955  Warning("Should use 2-D Smagorinsky for mesoscale resolution");
956  } else if (solverChoice.turbChoice[lev].les_type == LESType::Deardorff) {
957  Warning("Should not use Deardorff LES for mesoscale resolution");
958  }
959  }
960 
961  // Turn off implicit solve if we have no diffusion
962  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
963  bool l_use_diff = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
964  l_use_kturb );
965  bool l_implicit_diff = (solverChoice.vert_implicit_fac[0] > 0 ||
968  if (l_implicit_diff && !l_use_diff) {
969  Print() << "No molecular or turbulent diffusion, turning off implicit solve" << std::endl;
973  }
974  }
975 }
void init_bcs()
Definition: ERF_InitBCs.cpp:20

◆ initHSE() [1/2]

void ERF::initHSE ( )
private

Initialize HSE.

147 {
148  for (int lev = 0; lev <= finest_level; lev++)
149  {
150  initHSE(lev);
151  }
152 }

◆ initHSE() [2/2]

void ERF::initHSE ( int  lev)
private

Initialize density and pressure base state in hydrostatic equilibrium.

21 {
22  // This integrates up through column to update p_hse, pi_hse, th_hse;
23  // r_hse is not const b/c FillBoundary is called at the end for r_hse and p_hse
24 
25  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
26  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
27  MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
28  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
29  MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
30 
31  bool all_boxes_touch_bottom = true;
32  Box domain(geom[lev].Domain());
33 
34  int icomp = 0; int ncomp = BaseState::num_comps;
35 
36  if (lev == 0) {
37  BoxArray ba(base_state[lev].boxArray());
38  for (int i = 0; i < ba.size(); i++) {
39  if (ba[i].smallEnd(2) != domain.smallEnd(2)) {
40  all_boxes_touch_bottom = false;
41  }
42  }
43  }
44  else
45  {
46  //
47  // We need to do this interp from coarse level in order to set the values of
48  // the base state inside the domain but outside of the fine region
49  //
50  base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
51  //
52  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
53  // have been pre-filled - this includes ghost cells both inside and outside
54  // the domain
55  //
56  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
57  IntVect(0,0,0), // do not fill ghost cells outside the domain
58  base_state[lev-1], icomp, icomp, ncomp,
59  geom[lev-1], geom[lev],
60  refRatio(lev-1), &cell_cons_interp,
62 
63  // We need to do this here because the interpolation above may leave corners unfilled
64  // when the corners need to be filled by, for example, reflection of the fine ghost
65  // cell outside the fine region but inide the domain.
66  (*physbcs_base[lev])(base_state[lev],icomp,ncomp,base_state[lev].nGrowVect());
67  }
68 
69  if (all_boxes_touch_bottom || lev > 0) {
70 
71  // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
73  prob->erf_init_dens_hse_moist(r_hse, z_phys_nd[lev], geom[lev]);
74  } else {
75  prob->erf_init_dens_hse(r_hse, z_phys_nd[lev], z_phys_cc[lev], geom[lev]);
76  }
77 
78  erf_enforce_hse(lev, r_hse, p_hse, pi_hse, th_hse, qv_hse, z_phys_cc[lev]);
79 
80  //
81  // Impose physical bc's on the base state
82  //
83  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
84 
85  } else {
86 
87  BoxArray ba_new(domain);
88 
89  ChopGrids2D(ba_new, domain, ParallelDescriptor::NProcs());
90 
91  DistributionMapping dm_new(ba_new);
92 
93  MultiFab new_base_state(ba_new, dm_new, BaseState::num_comps, base_state[lev].nGrowVect());
94  new_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
95  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
96 
97  MultiFab new_r_hse (new_base_state, make_alias, BaseState::r0_comp, 1);
98  MultiFab new_p_hse (new_base_state, make_alias, BaseState::p0_comp, 1);
99  MultiFab new_pi_hse(new_base_state, make_alias, BaseState::pi0_comp, 1);
100  MultiFab new_th_hse(new_base_state, make_alias, BaseState::th0_comp, 1);
101  MultiFab new_qv_hse(new_base_state, make_alias, BaseState::qv0_comp, 1);
102 
103  std::unique_ptr<MultiFab> new_z_phys_cc;
104  std::unique_ptr<MultiFab> new_z_phys_nd;
105  if (solverChoice.mesh_type != MeshType::ConstantDz) {
106  new_z_phys_cc = std::make_unique<MultiFab>(ba_new,dm_new,1,1);
107  new_z_phys_cc->ParallelCopy(*z_phys_cc[lev],0,0,1,1,1);
108 
109  BoxArray ba_new_nd(ba_new);
110  ba_new_nd.surroundingNodes();
111  new_z_phys_nd = std::make_unique<MultiFab>(ba_new_nd,dm_new,1,1);
112  new_z_phys_nd->ParallelCopy(*z_phys_nd[lev],0,0,1,1,1);
113  }
114 
115  // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
117  prob->erf_init_dens_hse_moist(new_r_hse, new_z_phys_nd, geom[lev]);
118  } else {
119  prob->erf_init_dens_hse(new_r_hse, new_z_phys_nd, new_z_phys_cc, geom[lev]);
120  }
121 
122  erf_enforce_hse(lev, new_r_hse, new_p_hse, new_pi_hse, new_th_hse, new_qv_hse, new_z_phys_cc);
123 
124  //
125  // Impose physical bc's on the base state (we must make new, temporary bcs object because the z_phys_nd is different)
126  //
127  ERFPhysBCFunct_base* temp_physbcs_base =
128  new ERFPhysBCFunct_base(lev, geom[lev], domain_bcs_type, domain_bcs_type_d, new_z_phys_nd,
129  (solverChoice.terrain_type == TerrainType::MovingFittedMesh));
130  (*temp_physbcs_base)(new_base_state,0,new_base_state.nComp(),new_base_state.nGrowVect());
131  delete temp_physbcs_base;
132 
133  // Now copy back into the original arrays
134  base_state[lev].ParallelCopy(new_base_state,0,0,base_state[lev].nComp(),
135  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
136  }
137 
138  //
139  // Impose physical bc's on the base state -- the values outside the fine region
140  // but inside the domain have already been filled in the call above to InterpFromCoarseLevel
141  //
142  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
143 }
void ChopGrids2D(BoxArray &ba, const Box &domain, int target_size)
Definition: ERF_ChopGrids.cpp:21
Definition: ERF_PhysBCFunct.H:286
void erf_enforce_hse(int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, amrex::MultiFab &qv, std::unique_ptr< amrex::MultiFab > &z_cc)
Definition: ERF_Init1D.cpp:164
bool use_moist_background
Definition: ERF_DataStruct.H:1045
Here is the call graph for this function:

◆ initialize_integrator()

void ERF::initialize_integrator ( int  lev,
amrex::MultiFab &  cons_mf,
amrex::MultiFab &  vel_mf 
)
private
731 {
732  const BoxArray& ba(cons_mf.boxArray());
733  const DistributionMapping& dm(cons_mf.DistributionMap());
734 
735  int ncomp_cons = cons_mf.nComp();
736 
737  // Initialize the integrator memory
738  Vector<MultiFab> int_state; // integration state data structure example
739  int_state.push_back(MultiFab(cons_mf, make_alias, 0, ncomp_cons)); // cons
740  int_state.push_back(MultiFab(convert(ba,IntVect(1,0,0)), dm, 1, vel_mf.nGrow())); // xmom
741  int_state.push_back(MultiFab(convert(ba,IntVect(0,1,0)), dm, 1, vel_mf.nGrow())); // ymom
742  int_state.push_back(MultiFab(convert(ba,IntVect(0,0,1)), dm, 1, vel_mf.nGrow())); // zmom
743 
744  mri_integrator_mem[lev] = std::make_unique<MRISplitIntegrator<Vector<MultiFab> > >(int_state);
745  mri_integrator_mem[lev]->setNoSubstepping((solverChoice.substepping_type[lev] == SubsteppingType::None));
746  mri_integrator_mem[lev]->setAnelastic(solverChoice.anelastic[lev]);
747  mri_integrator_mem[lev]->setNcompCons(ncomp_cons);
748  mri_integrator_mem[lev]->setForceFirstStageSingleSubstep(solverChoice.force_stage1_single_substep);
749 }

◆ InitializeFromFile()

void ERF::InitializeFromFile ( )
private

◆ InitializeLevelFromData()

void ERF::InitializeLevelFromData ( int  lev,
const amrex::MultiFab &  initial_data 
)
private

◆ initializeMicrophysics()

void ERF::initializeMicrophysics ( const int &  a_nlevsmax)
private
Parameters
a_nlevsmaxnumber of AMR levels
1835 {
1836  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Eulerian) {
1837 
1838  micro = std::make_unique<EulerianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
1839 
1840  } else if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
1841 #ifdef ERF_USE_PARTICLES
1842 
1843  micro = std::make_unique<LagrangianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
1844  /* Lagrangian microphysics models will have a particle container; it needs to be added
1845  to ERF::particleData */
1846  const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
1847  /* The particle container has not yet been constructed and initialized, so just add
1848  its name here for now (so that functions to set plotting variables can see it). */
1849  particleData.addName( pc_name );
1850 
1851 #else
1852  Abort("Lagrangian microphysics can be used when compiled with ERF_USE_PARTICLES");
1853 #endif
1854  }
1855 
1856  qmoist.resize(a_nlevsmax);
1857  return;
1858 }
amrex::Vector< amrex::Vector< amrex::MultiFab * > > qmoist
Definition: ERF.H:857
Here is the call graph for this function:

◆ initRayleigh()

void ERF::initRayleigh ( )
private

Initialize Rayleigh damping profiles.

Initialization function for host and device vectors used to store averaged quantities when calculating the effects of Rayleigh Damping.

15 {
16  const int khi = geom[0].Domain().bigEnd(2);
17  solverChoice.dampingChoice.rayleigh_ztop = (solverChoice.terrain_type == TerrainType::None) ? geom[0].ProbHi(2) : zlevels_stag[0][khi+1];
18 
19  h_rayleigh_ptrs.resize(max_level+1);
20  d_rayleigh_ptrs.resize(max_level+1);
21 
22  h_sinesq_ptrs.resize(max_level+1);
23  d_sinesq_ptrs.resize(max_level+1);
24 
25  h_sinesq_stag_ptrs.resize(max_level+1);
26  d_sinesq_stag_ptrs.resize(max_level+1);
27 
28  for (int lev = 0; lev <= finest_level; lev++)
29  {
30  // These have 4 components: ubar, vbar, wbar, thetabar
31  h_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
32  d_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
33 
34  const int zlen_rayleigh = geom[lev].Domain().length(2);
35 
36  // Allocate space for these 1D vectors
37  for (int n = 0; n < Rayleigh::nvars; n++) {
38  h_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
39  d_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
40  }
41 
42  h_sinesq_ptrs[lev].resize(zlen_rayleigh);
43  d_sinesq_ptrs[lev].resize(zlen_rayleigh);
44 
45  h_sinesq_stag_ptrs[lev].resize(zlen_rayleigh+1);
46  d_sinesq_stag_ptrs[lev].resize(zlen_rayleigh+1);
47 
50 
51  for (int k = 0; k < zlen_rayleigh; k++) {
52  Real z = 0.5 * (zlevels_stag[lev][k] + zlevels_stag[lev][k+1]);
53  if (z > (ztop - zdamp)) {
54  Real zfrac = 1.0 - (ztop - z) / zdamp;
55  Real s = std::sin(PIoTwo*zfrac);
56  h_sinesq_ptrs[lev][k] = s*s;
57  } else {
58  h_sinesq_ptrs[lev][k] = 0.0;
59  }
60  }
61 
62  for (int k = 0; k < zlen_rayleigh+1; k++) {
63  Real z = zlevels_stag[lev][k];
64  if (z > (ztop - zdamp)) {
65  Real zfrac = 1.0 - (ztop - z) / zdamp;
66  Real s = std::sin(PIoTwo*zfrac);
67  h_sinesq_stag_ptrs[lev][k] = s*s;
68  } else {
69  h_sinesq_stag_ptrs[lev][k] = 0.0;
70  }
71  }
72 
73  // Init the host vectors for the reference states
74  prob->erf_init_rayleigh(h_rayleigh_ptrs[lev], geom[lev], z_phys_nd[lev], solverChoice.dampingChoice.rayleigh_zdamp);
75 
76  // Copy from host vectors to device vectors
77  for (int n = 0; n < Rayleigh::nvars; n++) {
78  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][n].begin(), h_rayleigh_ptrs[lev][n].end(),
79  d_rayleigh_ptrs[lev][n].begin());
80  }
81  Gpu::copy(Gpu::hostToDevice, h_sinesq_ptrs[lev].begin(), h_sinesq_ptrs[lev].end(), d_sinesq_ptrs[lev].begin());
82  Gpu::copy(Gpu::hostToDevice, h_sinesq_stag_ptrs[lev].begin(), h_sinesq_stag_ptrs[lev].end(), d_sinesq_stag_ptrs[lev].begin());
83  }
84 }
constexpr amrex::Real PIoTwo
Definition: ERF_Constants.H:7
amrex::Vector< amrex::Vector< amrex::Real > > h_sinesq_ptrs
Definition: ERF.H:1302
amrex::Vector< amrex::Vector< amrex::Real > > h_sinesq_stag_ptrs
Definition: ERF.H:1303
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_rayleigh_ptrs
Definition: ERF.H:1298
amrex::Real rayleigh_ztop
Definition: ERF_DampingStruct.H:79
amrex::Real rayleigh_zdamp
Definition: ERF_DampingStruct.H:78

◆ initSponge()

void ERF::initSponge ( )
private

Initialize sponge profiles.

Initialization function for host and device vectors used to store the effects of sponge Damping.

36 {
37  h_sponge_ptrs.resize(max_level+1);
38  d_sponge_ptrs.resize(max_level+1);
39 
40  for (int lev = 0; lev <= finest_level; lev++)
41  {
42  // These have 2 components: ubar, vbar
45 
46  const int zlen_sponge = geom[lev].Domain().length(2);
47 
48  // Allocate space for these 1D vectors
49  for (int n = 0; n < Sponge::nvars_sponge; n++) {
50  h_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
51  d_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
52  }
53 
54  }
55 }
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_sponge_ptrs
Definition: ERF.H:1299

◆ input_sponge()

void ERF::input_sponge ( int  lev)

High level wrapper for sponge x and y velocities level data from input sponge data.

Parameters
levInteger specifying the current level
18 {
19  // We only want to read the file once
20  if (lev == 0) {
22  Error("input_sounding file name must be provided via input");
23 
24  // this will interpolate the input profiles to the nominal height levels
25  // (ranging from 0 to the domain top)
27  }
28 }
InputSpongeData input_sponge_data
Definition: ERF.H:764
void read_from_file(const amrex::Geometry &geom, const amrex::Vector< amrex::Real > &zlevels_stag)
Definition: ERF_InputSpongeData.H:28
std::string input_sponge_file
Definition: ERF_InputSpongeData.H:108

◆ InterpWeatherDataOntoMesh()

void ERF::InterpWeatherDataOntoMesh ( const amrex::Geometry &  geom_weather,
amrex::MultiFab &  weather_forecast_interp,
amrex::Vector< amrex::Vector< amrex::MultiFab >> &  forecast_state 
)
271 {
272 
273  MultiFab& weather_mf = weather_forecast_data;
274  MultiFab& erf_mf_cons = forecast_state[0][Vars::cons];
275  MultiFab& erf_mf_xvel = forecast_state[0][Vars::xvel];
276  MultiFab& erf_mf_yvel = forecast_state[0][Vars::yvel];
277  MultiFab& erf_mf_zvel = forecast_state[0][Vars::zvel];
278  MultiFab& erf_mf_latlon = forecast_state[0][4];
279 
280  erf_mf_cons.setVal(0.0);
281  erf_mf_xvel.setVal(0.0);
282  erf_mf_yvel.setVal(0.0);
283  erf_mf_zvel.setVal(0.0);
284  erf_mf_latlon.setVal(0.0);
285 
286  BoxList bl_erf = erf_mf_cons.boxArray().boxList();
287  BoxList bl_weather = weather_mf.boxArray().boxList();
288 
289  const auto prob_lo_erf = geom[0].ProbLoArray();
290  const auto dx_erf = geom[0].CellSizeArray();
291 
292  for (auto& b : bl_erf) {
293  // You look at the lo corner of b, and find out the lowest cell in
294  // coarse weather data you need for the interpolation. That gives
295  // you the lo corner of the new b. Similarly, you can find out the
296  // hi corner of the new b. For cells outside the coarse_weath_data's
297  // bounding data, it's up to you. You probably want to use a biased
298  // interpolation stencil.
299 
300  // Get the cell indices of the bottom corner and top corner
301  const IntVect& lo_erf = b.smallEnd(); // Lower corner (inclusive)
302  const IntVect& hi_erf = b.bigEnd(); // Upper corner (inclusive)
303 
304  Real x = prob_lo_erf[0] + lo_erf[0] * dx_erf[0];
305  Real y = prob_lo_erf[1] + lo_erf[1] * dx_erf[1];
306  Real z = prob_lo_erf[2] + lo_erf[2] * dx_erf[2];
307 
308  auto idx_lo = find_bound_idx(x, y, z, bl_weather, geom_weather, BoundType::Lo);
309 
310  x = prob_lo_erf[0] + (hi_erf[0]+1) * dx_erf[0];
311  y = prob_lo_erf[1] + (hi_erf[1]+1) * dx_erf[1];
312  z = prob_lo_erf[2] + (hi_erf[2]+1) * dx_erf[2];
313 
314  auto idx_hi = find_bound_idx(x, y, z, bl_weather, geom_weather, BoundType::Hi);
315 
316  b.setSmall(idx_lo);
317  b.setBig(idx_hi);
318  }
319 
320  BoxArray cba(std::move(bl_erf));
321  cba.convert(IndexType::TheNodeType()); // <-- Make it nodal in all directions
322  MultiFab tmp_coarse_data(cba, erf_mf_cons.DistributionMap(), weather_mf.nComp(), 0);
323  tmp_coarse_data.ParallelCopy(weather_mf);
324 
325  //PlotMultiFab(weather_mf, geom_weather, "plt_coarse_weather_par_copy",MultiFabType::NC);
326 
327  const auto prob_lo_weather = geom_weather.ProbLoArray();
328  const auto dx_weather = geom_weather.CellSizeArray();
329 
330  for (MFIter mfi(erf_mf_cons); mfi.isValid(); ++mfi) {
331  const Array4<Real> &fine_cons_arr = erf_mf_cons.array(mfi);
332  const Array4<Real> &fine_xvel_arr = erf_mf_xvel.array(mfi);
333  const Array4<Real> &fine_yvel_arr = erf_mf_yvel.array(mfi);
334  //const Array4<Real> &fine_zvel_arr = erf_mf_zvel.array(mfi);
335  const Array4<Real> &fine_latlon_arr = erf_mf_latlon.array(mfi);
336 
337  const Array4<Real> &crse_arr = tmp_coarse_data.array(mfi);
338 
339  const Box& gbx = mfi.growntilebox(); // tilebox + ghost cells
340 
341  const Box &gtbx = mfi.tilebox(IntVect(1,0,0));
342  const Box &gtby = mfi.tilebox(IntVect(0,1,0));
343  //const Box &gtbz = mfi.tilebox(IntVect(0,0,1));
344 
345  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
346  // Physical location of the fine node
347  Real x = prob_lo_erf[0] + (i+0.5) * dx_erf[0];
348  Real y = prob_lo_erf[1] + (j+0.5) * dx_erf[1];
349  Real z = prob_lo_erf[2] + (k+0.5) * dx_erf[2];
350 
351  Real rho = interpolate_from_coarse(crse_arr, 0, x, y, z, prob_lo_weather.data(), dx_weather.data());
352  Real lat = interpolate_from_coarse(crse_arr, 8, x, y, z, prob_lo_weather.data(), dx_weather.data());
353  Real lon = interpolate_from_coarse(crse_arr, 9, x, y, z, prob_lo_weather.data(), dx_weather.data());
354 
355  fine_cons_arr(i,j,k,Rho_comp) = rho;
356 
357  fine_latlon_arr(i,j,k,0) = lat;
358  fine_latlon_arr(i,j,k,1) = lon;
359  });
360 
361  ParallelFor(gtbx, gtby,
362  [=] AMREX_GPU_DEVICE(int i, int j, int k) {
363  // Physical location of the fine node
364  Real x = prob_lo_erf[0] + i * dx_erf[0];
365  Real y = prob_lo_erf[1] + (j+0.5) * dx_erf[1];
366  Real z = prob_lo_erf[2] + (k+0.5) * dx_erf[2];
367  fine_xvel_arr(i, j, k, 0) = interpolate_from_coarse(crse_arr, 1, x, y, z, prob_lo_weather.data(), dx_weather.data());
368  },
369  [=] AMREX_GPU_DEVICE(int i, int j, int k) {
370  // Physical location of the fine node
371  Real x = prob_lo_erf[0] + (i+0.5) * dx_erf[0];
372  Real y = prob_lo_erf[1] + j * dx_erf[1];
373  Real z = prob_lo_erf[2] + (k+0.5) * dx_erf[2];
374  fine_yvel_arr(i, j, k, 0) = interpolate_from_coarse(crse_arr, 2, x, y, z, prob_lo_weather.data(), dx_weather.data());
375  });
376  }
377 
378  /*Vector<std::string> varnames = {
379  "rho", "uvel", "vvel", "wvel", "theta", "qv", "qc", "qr"
380  }; // Customize variable names
381 
382  Vector<std::string> varnames_cons = {
383  "rho", "rhotheta", "ke", "sc", "rhoqv", "rhoqc", "rhoqr"
384  }; // Customize variable names
385 
386  Vector<std::string> varnames_plot_mf = {
387  "rho", "rhotheta", "rhoqv", "rhoqc", "rhoqr", "xvel", "yvel", "zvel", "latitude", "longitude"
388  }; // Customize variable names
389 
390 
391  const Real time = 0.0;
392 
393  std::string pltname = "plt_interp";
394 
395  MultiFab plot_mf(erf_mf_cons.boxArray(), erf_mf_cons.DistributionMap(),
396  10, 0);
397 
398  plot_mf.setVal(0.0);
399 
400  for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
401  const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
402  const Array4<Real> &erf_mf_cons_arr = erf_mf_cons.array(mfi);
403  const Array4<Real> &erf_mf_xvel_arr = erf_mf_xvel.array(mfi);
404  const Array4<Real> &erf_mf_yvel_arr = erf_mf_yvel.array(mfi);
405  const Array4<Real> &erf_mf_zvel_arr = erf_mf_zvel.array(mfi);
406  const Array4<Real> &erf_mf_latlon_arr = erf_mf_latlon.array(mfi);
407 
408  const Box& bx = mfi.validbox();
409 
410  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
411  plot_mf_arr(i,j,k,0) = erf_mf_cons_arr(i,j,k,Rho_comp);
412  plot_mf_arr(i,j,k,1) = erf_mf_cons_arr(i,j,k,RhoTheta_comp);
413  plot_mf_arr(i,j,k,2) = erf_mf_cons_arr(i,j,k,RhoQ1_comp);
414  plot_mf_arr(i,j,k,3) = erf_mf_cons_arr(i,j,k,RhoQ2_comp);
415  plot_mf_arr(i,j,k,4) = erf_mf_cons_arr(i,j,k,RhoQ3_comp);
416 
417  plot_mf_arr(i,j,k,5) = (erf_mf_xvel_arr(i,j,k,0) + erf_mf_xvel_arr(i+1,j,k,0))/2.0;
418  plot_mf_arr(i,j,k,6) = (erf_mf_yvel_arr(i,j,k,0) + erf_mf_yvel_arr(i,j+1,k,0))/2.0;
419  plot_mf_arr(i,j,k,7) = (erf_mf_zvel_arr(i,j,k,0) + erf_mf_zvel_arr(i,j,k+1,0))/2.0;
420 
421  plot_mf_arr(i,j,k,8) = erf_mf_latlon_arr(i,j,k,0);
422  plot_mf_arr(i,j,k,9) = erf_mf_latlon_arr(i,j,k,1);
423  });
424  }
425 
426 
427  WriteSingleLevelPlotfile(
428  pltname,
429  plot_mf,
430  varnames_plot_mf,
431  geom[0],
432  time,
433  0 // level
434  );*/
435 }
AMREX_GPU_DEVICE amrex::Real interpolate_from_coarse(const amrex::Array4< const amrex::Real > &crse, int n, amrex::Real x, amrex::Real y, amrex::Real z, const amrex::Real *prob_lo_crse, const amrex::Real *dx_crse)
Definition: ERF_Interpolation_Bilinear.H:77
IntVect find_bound_idx(const Real &x, const Real &y, const Real &z, const BoxList &bl_weather, const Geometry &geom_weather, BoundType bound_type)
Definition: ERF_WeatherDataInterpolation.cpp:217
Here is the call graph for this function:

◆ is_it_time_for_action()

bool ERF::is_it_time_for_action ( int  nstep,
amrex::Real  time,
amrex::Real  dt,
int  action_interval,
amrex::Real  action_per 
)
static

Helper function which uses the current step number, time, and timestep to determine whether it is time to take an action specified at every interval of timesteps.

Parameters
nstepTimestep number
timeCurrent time
dtlevTimestep for the current level
action_intervalInterval in number of timesteps for taking action
action_perInterval in simulation time for taking action
654 {
655  bool int_test = (action_interval > 0 && nstep % action_interval == 0);
656 
657  bool per_test = false;
658  if (action_per > 0.0) {
659  const int num_per_old = static_cast<int>(amrex::Math::floor((time - dtlev) / action_per));
660  const int num_per_new = static_cast<int>(amrex::Math::floor((time) / action_per));
661 
662  if (num_per_old != num_per_new) {
663  per_test = true;
664  }
665  }
666 
667  return int_test || per_test;
668 }

◆ make_eb_box()

void ERF::make_eb_box ( )

◆ make_eb_regular()

void ERF::make_eb_regular ( )

◆ make_physbcs()

void ERF::make_physbcs ( int  lev)
private
753 {
754  if (SolverChoice::mesh_type == MeshType::VariableDz) {
755  AMREX_ALWAYS_ASSERT(z_phys_nd[lev] != nullptr);
756  }
757 
758  physbcs_cons[lev] = std::make_unique<ERFPhysBCFunct_cons> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
760  z_phys_nd[lev], solverChoice.use_real_bcs, th_bc_data[lev].data());
761  physbcs_u[lev] = std::make_unique<ERFPhysBCFunct_u> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
763  z_phys_nd[lev], solverChoice.use_real_bcs, xvel_bc_data[lev].data());
764  physbcs_v[lev] = std::make_unique<ERFPhysBCFunct_v> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
766  z_phys_nd[lev], solverChoice.use_real_bcs, yvel_bc_data[lev].data());
767  physbcs_w[lev] = std::make_unique<ERFPhysBCFunct_w> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
770  solverChoice.use_real_bcs, zvel_bc_data[lev].data());
771  physbcs_base[lev] = std::make_unique<ERFPhysBCFunct_base> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d, z_phys_nd[lev],
772  (solverChoice.terrain_type == TerrainType::MovingFittedMesh));
773 }

◆ make_subdomains()

void ERF::make_subdomains ( const amrex::BoxList &  ba,
amrex::Vector< amrex::BoxArray > &  bins 
)
7 {
8  Vector<BoxList> bins_bl;
9 
10  // Clear out any old bins
11  bins.clear();
12 
13  // Iterate over boxes
14  for (auto bx : bl)
15  {
16  bool added = false;
17 
18  // Try to add box to existing bin
19  for (int j = 0; j < bins_bl.size(); ++j) {
20  BoxList& bin = bins_bl[j];
21  bool touches = false;
22 
23  for (auto& b : bin)
24  {
25  Box gbx(bx); gbx.grow(1);
26  if (gbx.intersects(b)) {
27  touches = true;
28  break;
29  }
30  }
31 
32  if (touches) {
33  bin.push_back(bx);
34  added = true;
35  break;
36  }
37  }
38 
39  // If box couldn't be added to existing bin, create new bin
40  if (!added) {
41  BoxList new_bin;
42  new_bin.push_back(bx);
43  bins_bl.push_back(new_bin);
44  }
45  }
46 
47  // Convert the BoxLists to BoxArrays
48  for (int i = 0; i < bins_bl.size(); ++i) {
49  bins.push_back(BoxArray(bins_bl[i]));
50  }
51 }

◆ MakeDiagnosticAverage()

void ERF::MakeDiagnosticAverage ( amrex::Vector< amrex::Real > &  h_havg,
amrex::MultiFab &  S,
int  n 
)
2644 {
2645  // Get the number of cells in z at level 0
2646  int dir_z = AMREX_SPACEDIM-1;
2647  auto domain = geom[0].Domain();
2648  int size_z = domain.length(dir_z);
2649  int start_z = domain.smallEnd()[dir_z];
2650  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
2651 
2652  // resize the level 0 horizontal average vectors
2653  h_havg.resize(size_z, 0.0_rt);
2654 
2655  // Get the cell centered data and construct sums
2656 #ifdef _OPENMP
2657 #pragma omp parallel if (Gpu::notInLaunchRegion())
2658 #endif
2659  for (MFIter mfi(S); mfi.isValid(); ++mfi) {
2660  const Box& box = mfi.validbox();
2661  const IntVect& se = box.smallEnd();
2662  const IntVect& be = box.bigEnd();
2663 
2664  auto fab_arr = S[mfi].array();
2665 
2666  FArrayBox fab_reduce(box, 1, The_Async_Arena());
2667  auto arr_reduce = fab_reduce.array();
2668 
2669  ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
2670  arr_reduce(i, j, k, 0) = fab_arr(i,j,k,n);
2671  });
2672 
2673  for (int k=se[dir_z]; k <= be[dir_z]; ++k) {
2674  Box kbox(box); kbox.setSmall(dir_z,k); kbox.setBig(dir_z,k);
2675  h_havg[k-start_z] += fab_reduce.sum<RunOn::Device>(kbox,0);
2676  }
2677  }
2678 
2679  // combine sums from different MPI ranks
2680  ParallelDescriptor::ReduceRealSum(h_havg.dataPtr(), h_havg.size());
2681 
2682  // divide by the total number of cells we are averaging over
2683  for (int k = 0; k < size_z; ++k) {
2684  h_havg[k] /= area_z;
2685  }
2686 }

◆ MakeEBGeometry()

void ERF::MakeEBGeometry ( )

◆ MakeFilename_EyeTracker_latlon()

std::string ERF::MakeFilename_EyeTracker_latlon ( int  nstep)
631  {
632  // Ensure output directory exists
633  const std::string dir = "Output_HurricaneTracker/latlon";
634  if (!fs::exists(dir)) {
635  fs::create_directories(dir);
636  }
637 
638  // Construct filename with zero-padded step
639  std::ostringstream oss;
640  if(nstep==0){
641  oss << dir << "/hurricane_track_latlon" << std::setw(7) << std::setfill('0') << nstep << ".txt";
642  } else {
643  oss << dir << "/hurricane_track_latlon" << std::setw(7) << std::setfill('0') << nstep+1 << ".txt";
644  }
645 
646  return oss.str();
647 }

◆ MakeFilename_EyeTracker_maxvel()

std::string ERF::MakeFilename_EyeTracker_maxvel ( int  nstep)
650  {
651  // Ensure output directory exists
652  const std::string dir = "Output_HurricaneTracker/maxvel";
653  if (!fs::exists(dir)) {
654  fs::create_directories(dir);
655  }
656 
657  // Construct filename with zero-padded step
658  std::ostringstream oss;
659  if(nstep==0){
660  oss << dir << "/hurricane_maxvel_" << std::setw(7) << std::setfill('0') << nstep << ".txt";
661  } else {
662  oss << dir << "/hurricane_maxvel_" << std::setw(7) << std::setfill('0') << nstep+1 << ".txt";
663  }
664 
665  return oss.str();
666 }

◆ MakeHorizontalAverages()

void ERF::MakeHorizontalAverages ( )
2538 {
2539  int lev = 0;
2540 
2541  // First, average down all levels (if doing two-way coupling)
2542  if (solverChoice.coupling_type == CouplingType::TwoWay) {
2543  AverageDown();
2544  }
2545 
2546  MultiFab mf(grids[lev], dmap[lev], 5, 0);
2547 
2548  int zdir = 2;
2549  auto domain = geom[0].Domain();
2550 
2551  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
2552  bool is_anelastic = (solverChoice.anelastic[lev] == 1);
2553 
2554  for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
2555  const Box& bx = mfi.validbox();
2556  auto fab_arr = mf.array(mfi);
2557  auto const hse_arr = base_state[lev].const_array(mfi);
2558  auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
2559  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
2560  Real dens = cons_arr(i, j, k, Rho_comp);
2561  fab_arr(i, j, k, 0) = dens;
2562  fab_arr(i, j, k, 1) = cons_arr(i, j, k, RhoTheta_comp) / dens;
2563  if (!use_moisture) {
2564  if (is_anelastic) {
2565  fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
2566  } else {
2567  fab_arr(i,j,k,2) = getPgivenRTh(cons_arr(i,j,k,RhoTheta_comp));
2568  }
2569  }
2570  });
2571  }
2572 
2573  if (use_moisture)
2574  {
2575  for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
2576  const Box& bx = mfi.validbox();
2577  auto fab_arr = mf.array(mfi);
2578  auto const hse_arr = base_state[lev].const_array(mfi);
2579  auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
2580  int ncomp = vars_new[lev][Vars::cons].nComp();
2581 
2582  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
2583  Real dens = cons_arr(i, j, k, Rho_comp);
2584  if (is_anelastic) {
2585  fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
2586  } else {
2587  Real qv = cons_arr(i, j, k, RhoQ1_comp) / dens;
2588  fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
2589  }
2590  fab_arr(i, j, k, 3) = (ncomp > RhoQ1_comp ? cons_arr(i, j, k, RhoQ1_comp) / dens : 0.0);
2591  fab_arr(i, j, k, 4) = (ncomp > RhoQ2_comp ? cons_arr(i, j, k, RhoQ2_comp) / dens : 0.0);
2592  });
2593  }
2594 
2595  Gpu::HostVector<Real> h_avg_qv = sumToLine(mf,3,1,domain,zdir);
2596  Gpu::HostVector<Real> h_avg_qc = sumToLine(mf,4,1,domain,zdir);
2597  }
2598 
2599  // Sum in the horizontal plane
2600  Gpu::HostVector<Real> h_avg_density = sumToLine(mf,0,1,domain,zdir);
2601  Gpu::HostVector<Real> h_avg_temperature = sumToLine(mf,1,1,domain,zdir);
2602  Gpu::HostVector<Real> h_avg_pressure = sumToLine(mf,2,1,domain,zdir);
2603 
2604  // Divide by the total number of cells we are averaging over
2605  int size_z = domain.length(zdir);
2606  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
2607  int klen = static_cast<int>(h_avg_density.size());
2608 
2609  for (int k = 0; k < klen; ++k) {
2610  h_havg_density[k] /= area_z;
2611  h_havg_temperature[k] /= area_z;
2612  h_havg_pressure[k] /= area_z;
2613  if (solverChoice.moisture_type != MoistureType::None)
2614  {
2615  h_havg_qc[k] /= area_z;
2616  h_havg_qv[k] /= area_z;
2617  }
2618  } // k
2619 
2620  // resize device vectors
2621  d_havg_density.resize(size_z, 0.0_rt);
2622  d_havg_temperature.resize(size_z, 0.0_rt);
2623  d_havg_pressure.resize(size_z, 0.0_rt);
2624 
2625  // copy host vectors to device vectors
2626  Gpu::copy(Gpu::hostToDevice, h_havg_density.begin(), h_havg_density.end(), d_havg_density.begin());
2627  Gpu::copy(Gpu::hostToDevice, h_havg_temperature.begin(), h_havg_temperature.end(), d_havg_temperature.begin());
2628  Gpu::copy(Gpu::hostToDevice, h_havg_pressure.begin(), h_havg_pressure.end(), d_havg_pressure.begin());
2629 
2630  if (solverChoice.moisture_type != MoistureType::None)
2631  {
2632  d_havg_qv.resize(size_z, 0.0_rt);
2633  d_havg_qc.resize(size_z, 0.0_rt);
2634  Gpu::copy(Gpu::hostToDevice, h_havg_qv.begin(), h_havg_qv.end(), d_havg_qv.begin());
2635  Gpu::copy(Gpu::hostToDevice, h_havg_qc.begin(), h_havg_qc.end(), d_havg_qc.begin());
2636  }
2637 }
amrex::Gpu::DeviceVector< amrex::Real > d_havg_temperature
Definition: ERF.H:1320
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qv
Definition: ERF.H:1322
amrex::Vector< amrex::Real > h_havg_pressure
Definition: ERF.H:1315
amrex::Vector< amrex::Real > h_havg_qc
Definition: ERF.H:1317
amrex::Vector< amrex::Real > h_havg_density
Definition: ERF.H:1313
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qc
Definition: ERF.H:1323
amrex::Gpu::DeviceVector< amrex::Real > d_havg_density
Definition: ERF.H:1319
amrex::Vector< amrex::Real > h_havg_temperature
Definition: ERF.H:1314
amrex::Gpu::DeviceVector< amrex::Real > d_havg_pressure
Definition: ERF.H:1321
amrex::Vector< amrex::Real > h_havg_qv
Definition: ERF.H:1316
Here is the call graph for this function:

◆ MakeNewLevelFromCoarse()

void ERF::MakeNewLevelFromCoarse ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
291 {
292  AMREX_ALWAYS_ASSERT(lev > 0);
293 
294  if (verbose) {
295  amrex::Print() <<" NEW BA FROM COARSE AT LEVEL " << lev << " " << ba << std::endl;
296  }
297 
298  //
299  // Grow the subdomains vector and build the subdomains vector at this level
300  //
301  subdomains.resize(lev+1);
302  //
303  // Create subdomains at each level within the domain such that
304  // 1) all boxes in a given subdomain are "connected"
305  // 2) no boxes in a subdomain touch any boxes in any other subdomain
306  //
308  BoxArray dom(geom[lev].Domain());
309  subdomains[lev].push_back(dom);
310  } else {
311  make_subdomains(ba.simplified_list(), subdomains[lev]);
312  }
313 
314  if (lev == 0) init_bcs();
315 
316  //********************************************************************************************
317  // This allocates all kinds of things, including but not limited to: solution arrays,
318  // terrain arrays, metric terms and base state.
319  // *******************************************************************************************
320  init_stuff(lev, ba, dm, vars_new[lev], vars_old[lev], base_state[lev], z_phys_nd[lev]);
321 
322  t_new[lev] = time;
323  t_old[lev] = time - 1.e200;
324 
325  // ********************************************************************************************
326  // Build the data structures for metric quantities used with terrain-fitted coordinates
327  // ********************************************************************************************
328  if ( solverChoice.terrain_type == TerrainType::EB ||
329  solverChoice.terrain_type == TerrainType::ImmersedForcing)
330  {
331  const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
332  const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
333  if (solverChoice.terrain_type == TerrainType::EB) {
334  eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
335  } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
336  eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
337  }
338  }
339  init_zphys(lev, time);
341 
342  //
343  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
344  //
345  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
346  for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
347  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
348  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
349  }
350  }
351 
352  // ********************************************************************************************
353  // Build the data structures for canopy model (depends upon z_phys)
354  // ********************************************************************************************
356  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
357  }
358 
359  //********************************************************************************************
360  // Microphysics
361  // *******************************************************************************************
362  int q_size = micro->Get_Qmoist_Size(lev);
363  qmoist[lev].resize(q_size);
364  micro->Define(lev, solverChoice);
365  if (solverChoice.moisture_type != MoistureType::None)
366  {
367  micro->Init(lev, vars_new[lev][Vars::cons],
368  grids[lev], Geom(lev), 0.0,
369  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
370  }
371  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
372  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
373  }
374 
375  //********************************************************************************************
376  // Radiation
377  // *******************************************************************************************
378  if (solverChoice.rad_type != RadiationType::None)
379  {
380  rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
381  }
382 
383  // *****************************************************************************************************
384  // Initialize the boundary conditions (after initializing the terrain but before calling
385  // initHSE or FillCoarsePatch)
386  // *****************************************************************************************************
387  make_physbcs(lev);
388 
389  // ********************************************************************************************
390  // Update the base state at this level by interpolation from coarser level
391  // ********************************************************************************************
392  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
393  IntVect(0,0,0), // do not fill ghost cells outside the domain
394  base_state[lev-1], 0, 0, base_state[lev].nComp(),
395  geom[lev-1], geom[lev],
396  refRatio(lev-1), &cell_cons_interp,
398 
399  // Impose bc's outside the domain
400  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
401 
402  // ********************************************************************************************
403  // Build the data structures for calculating diffusive/turbulent terms
404  // ********************************************************************************************
405  update_diffusive_arrays(lev, ba, dm);
406 
407  // ********************************************************************************************
408  // Fill data at the new level by interpolation from the coarser level
409  // Note that internal to FillCoarsePatch we will convert velocity to momentum,
410  // then interpolate momentum, then convert momentum back to velocity
411  // Also note that FillCoarsePatch is hard-wired to act only on lev_new at coarse and fine
412  // ********************************************************************************************
413  FillCoarsePatch(lev, time);
414 
415  // ********************************************************************************************
416  // Initialize the integrator class
417  // ********************************************************************************************
418  dt_mri_ratio[lev] = dt_mri_ratio[lev-1];
420 
421  // ********************************************************************************************
422  // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
423  // ********************************************************************************************
424  if (lev > 0 && cf_width >= 0) {
427  }
428 
429  // ********************************************************************************************
430  // Build the data structures for holding sea surface temps and skin temps
431  // ********************************************************************************************
432  sst_lev[lev].resize(1); sst_lev[lev][0] = nullptr;
433  tsk_lev[lev].resize(1); tsk_lev[lev][0] = nullptr;
434 
435  //********************************************************************************************
436  // Land Surface Model
437  // *******************************************************************************************
438  int lsm_data_size = lsm.Get_Data_Size();
439  int lsm_flux_size = lsm.Get_Flux_Size();
440  lsm_data[lev].resize(lsm_data_size);
441  lsm_data_name.resize(lsm_data_size);
442  lsm_flux[lev].resize(lsm_flux_size);
443  lsm_flux_name.resize(lsm_flux_size);
444  lsm.Define(lev, solverChoice);
445  if (solverChoice.lsm_type != LandSurfaceType::None)
446  {
447  lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), 0.0); // dummy dt value
448  }
449  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
450  lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
451  lsm_data_name[mvar] = lsm.Get_DataName(mvar);
452  }
453  for (int mvar(0); mvar<lsm_flux[lev].size(); ++mvar) {
454  lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
455  lsm_flux_name[mvar] = lsm.Get_FluxName(mvar);
456  }
457 
458  // ********************************************************************************************
459  // Create the SurfaceLayer arrays at this (new) level
460  // ********************************************************************************************
461  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
462  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
463  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
464  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,lev+1,
465  mfv_old, Theta_prim[lev], Qv_prim[lev],
466  Qr_prim[lev], z_phys_nd[lev],
467  Hwave[lev].get(), Lwave[lev].get(), eddyDiffs_lev[lev].get(),
469  sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
470  }
471 
472 #ifdef ERF_USE_PARTICLES
473  // particleData.Redistribute();
474 #endif
475 }
void update_diffusive_arrays(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
Definition: ERF_MakeNewArrays.cpp:474
void initialize_integrator(int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)
Definition: ERF_MakeNewArrays.cpp:730
void make_subdomains(const amrex::BoxList &ba, amrex::Vector< amrex::BoxArray > &bins)
Definition: ERF_MakeSubdomains.cpp:6
void update_terrain_arrays(int lev)
Definition: ERF_MakeNewArrays.cpp:713
void init_zphys(int lev, amrex::Real time)
Definition: ERF_MakeNewArrays.cpp:599
void init_stuff(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)
Definition: ERF_MakeNewArrays.cpp:24
void Define_ERFFillPatchers(int lev)
Definition: ERF.cpp:2715
int Get_Data_Size()
Definition: ERF_LandSurface.H:98
std::string Get_DataName(const int &varIdx)
Definition: ERF_LandSurface.H:104
std::string Get_FluxName(const int &varIdx)
Definition: ERF_LandSurface.H:110
amrex::MultiFab * Get_Flux_Ptr(const int &lev, const int &varIdx)
Definition: ERF_LandSurface.H:92
void Init(const int &lev, const amrex::MultiFab &cons_in, const amrex::Geometry &geom, const amrex::Real &dt_advance)
Definition: ERF_LandSurface.H:43
void Define(const int &lev, SolverChoice &sc)
Definition: ERF_LandSurface.H:36
int Get_Flux_Size()
Definition: ERF_LandSurface.H:101

◆ MakeNewLevelFromScratch()

void ERF::MakeNewLevelFromScratch ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
27 {
28  BoxArray ba;
29  DistributionMapping dm;
30  Box domain(Geom(0).Domain());
31  if (lev == 0 && restart_chkfile.empty() &&
32  (max_grid_size[0][0] >= domain.length(0)) &&
33  (max_grid_size[0][1] >= domain.length(1)) &&
34  ba_in.size() != ParallelDescriptor::NProcs())
35  {
36  // We only decompose in z if max_grid_size_z indicates we should
37  bool decompose_in_z = (max_grid_size[0][2] < domain.length(2));
38 
39  ba = ERFPostProcessBaseGrids(Geom(0).Domain(),decompose_in_z);
40  dm = DistributionMapping(ba);
41  } else {
42  ba = ba_in;
43  dm = dm_in;
44  }
45 
46  // ********************************************************************************************
47  // Define grids[lev] to be ba
48  // ********************************************************************************************
49  SetBoxArray(lev, ba);
50 
51  // ********************************************************************************************
52  // Define dmap[lev] to be dm
53  // ********************************************************************************************
54  SetDistributionMap(lev, dm);
55 
56  if (verbose) {
57  amrex::Print() << "BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl;
58  // amrex::Print() <<" SIMPLIFIED BA FROM SCRATCH AT LEVEL " << lev << " " << ba.simplified_list() << std::endl;
59  }
60 
61  subdomains.resize(lev+1);
62  if ( (lev == 0) || (
64  (solverChoice.init_type != InitType::WRFInput) && (solverChoice.init_type != InitType::Metgrid) ) ) {
65  BoxArray dom(geom[lev].Domain());
66  subdomains[lev].push_back(dom);
67  } else {
68  //
69  // Create subdomains at each level within the domain such that
70  // 1) all boxes in a given subdomain are "connected"
71  // 2) no boxes in a subdomain touch any boxes in any other subdomain
72  //
73  make_subdomains(ba.simplified_list(), subdomains[lev]);
74  }
75 
76  if (lev == 0) init_bcs();
77 
78  if ( solverChoice.terrain_type == TerrainType::EB ||
79  solverChoice.terrain_type == TerrainType::ImmersedForcing)
80  {
81  const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
82  const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
83  if (solverChoice.terrain_type == TerrainType::EB) {
84  eb[lev]->make_all_factories(lev, geom[lev], grids[lev], dmap[lev], eb_level);
85  } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
86  eb[lev]->make_cc_factory(lev, geom[lev], grids[lev], dmap[lev], eb_level);
87  }
88  } else {
89  // m_factory[lev] = std::make_unique<FabFactory<FArrayBox>>();
90  }
91 
92  auto& lev_new = vars_new[lev];
93  auto& lev_old = vars_old[lev];
94 
95  //********************************************************************************************
96  // This allocates all kinds of things, including but not limited to: solution arrays,
97  // terrain arrays, metric terms and base state.
98  // *******************************************************************************************
99  init_stuff(lev, ba, dm, lev_new, lev_old, base_state[lev], z_phys_nd[lev]);
100 
101  //********************************************************************************************
102  // Land Surface Model
103  // *******************************************************************************************
104  int lsm_data_size = lsm.Get_Data_Size();
105  int lsm_flux_size = lsm.Get_Flux_Size();
106  lsm_data[lev].resize(lsm_data_size);
107  lsm_data_name.resize(lsm_data_size);
108  lsm_flux[lev].resize(lsm_flux_size);
109  lsm_flux_name.resize(lsm_flux_size);
110  lsm.Define(lev, solverChoice);
111  if (solverChoice.lsm_type != LandSurfaceType::None)
112  {
113  lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), 0.0); // dummy dt value
114  }
115  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
116  lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
117  lsm_data_name[mvar] = lsm.Get_DataName(mvar);
118  }
119  for (int mvar(0); mvar<lsm_flux[lev].size(); ++mvar) {
120  lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
121  lsm_flux_name[mvar] = lsm.Get_FluxName(mvar);
122  }
123 
124 
125 
126  // ********************************************************************************************
127  // Build the data structures for calculating diffusive/turbulent terms
128  // ********************************************************************************************
129  update_diffusive_arrays(lev, ba, dm);
130 
131  // ********************************************************************************************
132  // Build the data structures for holding sea surface temps and skin temps
133  // ********************************************************************************************
134  sst_lev[lev].resize(1); sst_lev[lev][0] = nullptr;
135  tsk_lev[lev].resize(1); tsk_lev[lev][0] = nullptr;
136 
137  // ********************************************************************************************
138  // Thin immersed body
139  // *******************************************************************************************
140  init_thin_body(lev, ba, dm);
141 
142  // ********************************************************************************************
143  // Initialize the integrator class
144  // ********************************************************************************************
145  initialize_integrator(lev, lev_new[Vars::cons],lev_new[Vars::xvel]);
146 
147  // ********************************************************************************************
148  // Initialize the data itself
149  // If (init_type == InitType::WRFInput) then we are initializing terrain and the initial data in
150  // the same call so we must call init_only before update_terrain_arrays
151  // If (init_type != InitType::WRFInput) then we want to initialize the terrain before the initial data
152  // since we may need to use the grid information before constructing
153  // initial idealized data
154  // ********************************************************************************************
155  if (restart_chkfile.empty()) {
156  if ( (solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid) )
157  {
158  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type == TerrainType::StaticFittedMesh);
159  init_only(lev, time);
160  init_zphys(lev, time);
162  make_physbcs(lev);
163  } else {
164  init_zphys(lev, time);
166  // Note that for init_type != InitType::WRFInput and != InitType::Metgrid,
167  // make_physbcs is called inside init_only
168  init_only(lev, time);
169  }
170  } else {
171  // if restarting and nudging from input sounding, load the input sounding files
172  if (lev == 0 && solverChoice.init_type == InitType::Input_Sounding && solverChoice.nudging_from_input_sounding)
173  {
175  Error("input_sounding file name must be provided via input");
176  }
177 
179 
180  // this will interpolate the input profiles to the nominal height levels
181  // (ranging from 0 to the domain top)
182  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
183  input_sounding_data.read_from_file(geom[lev], zlevels_stag[lev], n);
184  }
185 
186  // this will calculate the hydrostatically balanced density and pressure
187  // profiles following WRF ideal.exe
188  if (solverChoice.sounding_type == SoundingType::Ideal) {
190  } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
191  solverChoice.sounding_type == SoundingType::DryIsentropic) {
192  input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
194  }
195  }
196 
197  // We re-create terrain_blanking on restart rather than storing it in the checkpoint
198  if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
199  int ngrow = ComputeGhostCells(solverChoice) + 2;
200  terrain_blanking[lev]->setVal(1.0);
201  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ngrow);
202  terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
203  }
204  }
205 
206  // Read in tables needed for windfarm simulations
207  // fill in Nturb multifab - number of turbines in each mesh cell
208  // write out the vtk files for wind turbine location and/or
209  // actuator disks
210  #ifdef ERF_USE_WINDFARM
211  init_windfarm(lev);
212  #endif
213 
214  // ********************************************************************************************
215  // Build the data structures for canopy model (depends upon z_phys)
216  // ********************************************************************************************
217  if (restart_chkfile.empty()) {
219  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
220  }
221  }
222 
223  //********************************************************************************************
224  // Create wall distance field for RANS model (depends upon z_phys)
225  // *******************************************************************************************
226  if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
227  // Handle bottom boundary
228  poisson_wall_dist(lev);
229 
230  // Correct the wall distance for immersed bodies
236  geom[lev],
237  z_phys_cc[lev]);
238  }
239  }
240 
241  //********************************************************************************************
242  // Microphysics
243  // *******************************************************************************************
244  int q_size = micro->Get_Qmoist_Size(lev);
245  qmoist[lev].resize(q_size);
246  micro->Define(lev, solverChoice);
247  if (solverChoice.moisture_type != MoistureType::None)
248  {
249  micro->Init(lev, vars_new[lev][Vars::cons],
250  grids[lev], Geom(lev), 0.0,
251  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
252  }
253  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
254  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
255  }
256 
257  //********************************************************************************************
258  // Radiation
259  // *******************************************************************************************
260  if (solverChoice.rad_type != RadiationType::None)
261  {
262  rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
263  }
264 
265  // ********************************************************************************************
266  // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
267  // ********************************************************************************************
268  if (lev > 0 && cf_width >= 0) {
271  }
272 
273 #ifdef ERF_USE_PARTICLES
274  if (restart_chkfile.empty()) {
275  if (lev == 0) {
276  initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,time);
277  } else {
278  particleData.Redistribute();
279  }
280  }
281 #endif
282 }
BoxArray ERFPostProcessBaseGrids(const Box &domain, bool decompose_in_z)
Definition: ERF_ChopGrids.cpp:6
void thinbody_wall_dist(std::unique_ptr< MultiFab > &wdist, Vector< IntVect > &xfaces, Vector< IntVect > &yfaces, Vector< IntVect > &zfaces, const Geometry &geomdata, std::unique_ptr< MultiFab > &z_phys_cc)
Definition: ERF_ThinBodyWallDist.cpp:12
void init_only(int lev, amrex::Real time)
Definition: ERF.cpp:1931
void poisson_wall_dist(int lev)
Definition: ERF_PoissonWallDist.cpp:20
void init_thin_body(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
Definition: ERF_MakeNewLevel.cpp:750
bool nudging_from_input_sounding
Definition: ERF_DataStruct.H:1005
Here is the call graph for this function:

◆ MakeVTKFilename()

std::string ERF::MakeVTKFilename ( int  nstep)
574  {
575  // Ensure output directory exists
576  const std::string dir = "Output_HurricaneTracker";
577  if (!fs::exists(dir)) {
578  fs::create_directory(dir);
579  }
580 
581  // Construct filename with zero-padded step
582  std::ostringstream oss;
583  if(nstep==0){
584  oss << dir << "/hurricane_track_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
585  } else {
586  oss << dir << "/hurricane_track_" << std::setw(7) << std::setfill('0') << nstep+1 << ".vtk";
587  }
588 
589  return oss.str();
590 }

◆ MakeVTKFilename_EyeTracker_xy()

std::string ERF::MakeVTKFilename_EyeTracker_xy ( int  nstep)
612  {
613  // Ensure output directory exists
614  const std::string dir = "Output_HurricaneTracker/xy";
615  if (!fs::exists(dir)) {
616  fs::create_directories(dir);
617  }
618 
619  // Construct filename with zero-padded step
620  std::ostringstream oss;
621  if(nstep==0){
622  oss << dir << "/hurricane_track_xy_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
623  } else {
624  oss << dir << "/hurricane_track_xy_" << std::setw(7) << std::setfill('0') << nstep+1 << ".vtk";
625  }
626 
627  return oss.str();
628 }

◆ MakeVTKFilename_TrackerCircle()

std::string ERF::MakeVTKFilename_TrackerCircle ( int  nstep)
593  {
594  // Ensure output directory exists
595  const std::string dir = "Output_HurricaneTracker/tracker_circle";
596  if (!fs::exists(dir)) {
597  fs::create_directories(dir);
598  }
599 
600  // Construct filename with zero-padded step
601  std::ostringstream oss;
602  if(nstep==0){
603  oss << dir << "/hurricane_tracker_circle_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
604  } else {
605  oss << dir << "/hurricane_tracker_circle_" << std::setw(7) << std::setfill('0') << nstep+1 << ".vtk";
606  }
607 
608  return oss.str();
609 }

◆ nghost_eb_basic()

static int ERF::nghost_eb_basic ( )
inlinestaticprivate
1633  { return 5; }

◆ nghost_eb_full()

static int ERF::nghost_eb_full ( )
inlinestaticprivate
1640  { return 4; }

◆ nghost_eb_volume()

static int ERF::nghost_eb_volume ( )
inlinestaticprivate
1637  { return 5; }

◆ NumDataLogs()

AMREX_FORCE_INLINE int ERF::NumDataLogs ( )
inlineprivatenoexcept
1429  {
1430  return datalog.size();
1431  }

◆ NumDerDataLogs()

AMREX_FORCE_INLINE int ERF::NumDerDataLogs ( )
inlineprivatenoexcept
1436  {
1437  return der_datalog.size();
1438  }

◆ NumSampleLineLogs()

AMREX_FORCE_INLINE int ERF::NumSampleLineLogs ( )
inlineprivatenoexcept
1465  {
1466  return samplelinelog.size();
1467  }

◆ NumSampleLines()

AMREX_FORCE_INLINE int ERF::NumSampleLines ( )
inlineprivatenoexcept
1491  {
1492  return sampleline.size();
1493  }

◆ NumSamplePointLogs()

AMREX_FORCE_INLINE int ERF::NumSamplePointLogs ( )
inlineprivatenoexcept
1451  {
1452  return sampleptlog.size();
1453  }

◆ NumSamplePoints()

AMREX_FORCE_INLINE int ERF::NumSamplePoints ( )
inlineprivatenoexcept
1478  {
1479  return samplepoint.size();
1480  }

◆ operator=() [1/2]

ERF& ERF::operator= ( const ERF other)
delete

◆ operator=() [2/2]

ERF& ERF::operator= ( ERF &&  other)
deletenoexcept

◆ ParameterSanityChecks()

void ERF::ParameterSanityChecks ( )
private
2471 {
2472  AMREX_ALWAYS_ASSERT(cfl > 0. || fixed_dt[0] > 0.);
2473 
2474  // We don't allow use_real_bcs to be true if init_type is not either InitType::WRFInput or InitType::Metgrid
2475  AMREX_ALWAYS_ASSERT( !solverChoice.use_real_bcs ||
2476  ((solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid)) );
2477 
2478  AMREX_ALWAYS_ASSERT(real_width >= 0);
2479  AMREX_ALWAYS_ASSERT(real_set_width >= 0);
2480  AMREX_ALWAYS_ASSERT(real_width >= real_set_width);
2481 
2482  if (cf_width < 0 || cf_set_width < 0 || cf_width < cf_set_width) {
2483  Abort("You must set cf_width >= cf_set_width >= 0");
2484  }
2485  if (max_level > 0 && cf_set_width > 0) {
2486  for (int lev = 1; lev <= max_level; lev++) {
2487  if (cf_set_width%ref_ratio[lev-1][0] != 0 ||
2488  cf_set_width%ref_ratio[lev-1][1] != 0 ||
2489  cf_set_width%ref_ratio[lev-1][2] != 0 ) {
2490  Abort("You must set cf_width to be a multiple of ref_ratio");
2491  }
2492  }
2493  }
2494 
2495  // If fixed_mri_dt_ratio is set, it must be even
2496  if (fixed_mri_dt_ratio > 0 && (fixed_mri_dt_ratio%2 != 0) )
2497  {
2498  Abort("If you specify fixed_mri_dt_ratio, it must be even");
2499  }
2500 
2501  for (int lev = 0; lev <= max_level; lev++)
2502  {
2503  // We ignore fixed_fast_dt if not substepping
2504  if (solverChoice.substepping_type[lev] == SubsteppingType::None) {
2505  fixed_fast_dt[lev] = -1.0;
2506  }
2507 
2508  // If both fixed_dt and fast_dt are specified, their ratio must be an even integer
2509  if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. && fixed_mri_dt_ratio <= 0)
2510  {
2511  Real eps = 1.e-12;
2512  int ratio = static_cast<int>( ( (1.0+eps) * fixed_dt[lev] ) / fixed_fast_dt[lev] );
2513  if (fixed_dt[lev] / fixed_fast_dt[lev] != ratio)
2514  {
2515  Abort("Ratio of fixed_dt to fixed_fast_dt must be an even integer");
2516  }
2517  }
2518 
2519  // If all three are specified, they must be consistent
2520  if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. && fixed_mri_dt_ratio > 0)
2521  {
2522  if (fixed_dt[lev] / fixed_fast_dt[lev] != fixed_mri_dt_ratio)
2523  {
2524  Abort("Dt is over-specfied");
2525  }
2526  }
2527  } // lev
2528 
2529  if (solverChoice.coupling_type == CouplingType::TwoWay && cf_width > 0) {
2530  Abort("For two-way coupling you must set cf_width = 0");
2531  }
2532 }
int real_set_width
Definition: ERF.H:1223

◆ PlotFileName()

std::string ERF::PlotFileName ( int  lev) const
private

◆ PlotFileVarNames()

Vector< std::string > ERF::PlotFileVarNames ( amrex::Vector< std::string >  plot_var_names)
staticprivate
296 {
297  Vector<std::string> names;
298 
299  names.insert(names.end(), plot_var_names.begin(), plot_var_names.end());
300 
301  return names;
302 
303 }

◆ poisson_wall_dist()

void ERF::poisson_wall_dist ( int  lev)

Calculate wall distances using the Poisson equation

The zlo boundary is assumed to correspond to the land surface. If there are no boundary walls, then the other use case is to calculate wall distances for immersed boundaries (embedded or thin body).

See Tucker, P. G. (2003). Differential equation-based wall distance computation for DES and RANS. Journal of Computational Physics, 190(1), 229–248. https://doi.org/10.1016/S0021-9991(03)00272-9

21 {
22  BL_PROFILE("ERF::poisson_wall_dist()");
23 
24  bool havewall{false};
25  Orientation zlo(Direction::z, Orientation::low);
26  if ( ( phys_bc_type[zlo] == ERF_BC::surface_layer ) ||
27  ( phys_bc_type[zlo] == ERF_BC::no_slip_wall ) )/*||
28  ((phys_bc_type[zlo] == ERF_BC::slip_wall) && (dom_hi.z > dom_lo.z)) )*/
29  {
30  havewall = true;
31  }
32 
33  auto const& geomdata = geom[lev];
34 
35  if (havewall) {
36  if (solverChoice.mesh_type == MeshType::ConstantDz) {
37 // Comment this out to test the wall dist calc in the trivial case:
38 //#if 0
39  Print() << "Directly calculating direct wall distance for constant dz" << std::endl;
40  const Real* prob_lo = geomdata.ProbLo();
41  const Real* dx = geomdata.CellSize();
42  for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
43  const Box& bx = mfi.validbox();
44  auto dist_arr = walldist[lev]->array(mfi);
45  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
46  dist_arr(i, j, k) = prob_lo[2] + (k + 0.5) * dx[2];
47  });
48  }
49  return;
50 //#endif
51  } else if (solverChoice.mesh_type == MeshType::StretchedDz) {
52  // TODO: Handle this trivial case
53  Error("Wall dist calc not implemented with grid stretching yet");
54  } else {
55  // TODO
56  Error("Wall dist calc not implemented over terrain yet");
57  }
58  }
59 
60  Print() << "Calculating Poisson wall distance" << std::endl;
61 
62  // Make sure the solver only sees the levels over which we are solving
63  BoxArray nba = walldist[lev]->boxArray();
64  nba.surroundingNodes();
65  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
66  Vector<BoxArray> ba_tmp; ba_tmp.push_back(nba);
67  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(walldist[lev]->DistributionMap());
68 
69  Vector<MultiFab> rhs;
70  Vector<MultiFab> phi;
71 
72  if (solverChoice.terrain_type == TerrainType::EB) {
73  amrex::Error("Wall dist calc not implemented for EB");
74  } else {
75  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
76  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
77  }
78 
79  rhs[0].setVal(-1.0);
80 
81  // Define an overset mask to set dirichlet nodes on walls
82  iMultiFab mask(ba_tmp[0], dm_tmp[0], 1, 0);
83  Vector<const iMultiFab*> overset_mask = {&mask};
84 
85  auto const dom_lo = lbound(geom[lev].Domain());
86  auto const dom_hi = ubound(geom[lev].Domain());
87 
88  // ****************************************************************************
89  // Initialize phi
90  // (It is essential that we do this in order to fill the corners; this is
91  // used if we include blanking.)
92  // ****************************************************************************
93  phi[0].setVal(0.0);
94 
95  // ****************************************************************************
96  // Interior boundaries are marked with phi=0
97  // ****************************************************************************
98  // Overset mask is 0/1: 1 means the node is an unknown. 0 means it's known.
99  mask.setVal(1);
101  Warning("Poisson distance is inaccurate for bodies in open domains that are small compared to the domain size, skipping");
102  return;
103 #if 0
104  Gpu::DeviceVector<IntVect> xfacelist, yfacelist, zfacelist;
105 
106  xfacelist.resize(solverChoice.advChoice.zero_xflux.size());
107  yfacelist.resize(solverChoice.advChoice.zero_yflux.size());
108  zfacelist.resize(solverChoice.advChoice.zero_zflux.size());
109 
110  if (xfacelist.size() > 0) {
111  Gpu::copy(amrex::Gpu::hostToDevice,
114  xfacelist.begin());
115  Print() << " masking interior xfaces" << std::endl;
116  }
117  if (yfacelist.size() > 0) {
118  Gpu::copy(amrex::Gpu::hostToDevice,
121  yfacelist.begin());
122  Print() << " masking interior yfaces" << std::endl;
123  }
124  if (zfacelist.size() > 0) {
125  Gpu::copy(amrex::Gpu::hostToDevice,
128  zfacelist.begin());
129  Print() << " masking interior zfaces" << std::endl;
130  }
131 
132  for (MFIter mfi(phi[0]); mfi.isValid(); ++mfi) {
133  const Box& bx = mfi.validbox();
134 
135  auto phi_arr = phi[0].array(mfi);
136  auto mask_arr = mask.array(mfi);
137 
138  if (xfacelist.size() > 0) {
139  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
140  for (int iface=0; iface < xfacelist.size(); ++iface) {
141  if ((i == xfacelist[iface][0]) &&
142  (j == xfacelist[iface][1]) &&
143  (k == xfacelist[iface][2]))
144  {
145  mask_arr(i, j , k ) = 0;
146  mask_arr(i, j , k+1) = 0;
147  mask_arr(i, j+1, k ) = 0;
148  mask_arr(i, j+1, k+1) = 0;
149  }
150  }
151  });
152  }
153 
154  if (yfacelist.size() > 0) {
155  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
156  for (int iface=0; iface < yfacelist.size(); ++iface) {
157  if ((i == yfacelist[iface][0]) &&
158  (j == yfacelist[iface][1]) &&
159  (k == yfacelist[iface][2]))
160  {
161  mask_arr(i , j, k ) = 0;
162  mask_arr(i , j, k+1) = 0;
163  mask_arr(i+1, j, k ) = 0;
164  mask_arr(i+1, j, k+1) = 0;
165  }
166  }
167  });
168  }
169 
170  if (zfacelist.size() > 0) {
171  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
172  for (int iface=0; iface < zfacelist.size(); ++iface) {
173  if ((i == xfacelist[iface][0]) &&
174  (j == xfacelist[iface][1]) &&
175  (k == xfacelist[iface][2]))
176  {
177  mask_arr(i , j , k) = 0;
178  mask_arr(i , j+1, k) = 0;
179  mask_arr(i+1, j , k) = 0;
180  mask_arr(i+1, j+1, k) = 0;
181  }
182  }
183  });
184  }
185  }
186 #endif
187  }
188 
189  // ****************************************************************************
190  // Setup BCs, with solid domain boundaries being dirichlet
191  // ****************************************************************************
192  amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> bc3d_lo, bc3d_hi;
193  for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
194  if (geom[0].isPeriodic(dir)) {
195  bc3d_lo[dir] = LinOpBCType::Periodic;
196  bc3d_hi[dir] = LinOpBCType::Periodic;
197  } else {
198  bc3d_lo[dir] = LinOpBCType::Neumann;
199  bc3d_hi[dir] = LinOpBCType::Neumann;
200  }
201  }
202  if (havewall) {
203  Print() << " Poisson zlo BC is dirichlet" << std::endl;
204  bc3d_lo[2] = LinOpBCType::Dirichlet;
205  }
206  Print() << " bc lo : " << bc3d_lo << std::endl;
207  Print() << " bc hi : " << bc3d_hi << std::endl;
208 
209  if (!solverChoice.advChoice.have_zero_flux_faces && !havewall) {
210  Error("No solid boundaries in the computational domain");
211  }
212 
213  LPInfo info;
214 /* Nodal solver cannot have hidden dimensions */
215 #if 0
216  // Allow a hidden direction if the domain is one cell wide
217  if (dom_lo.x == dom_hi.x) {
218  info.setHiddenDirection(0);
219  Print() << " domain is 2D in yz" << std::endl;
220  } else if (dom_lo.y == dom_hi.y) {
221  info.setHiddenDirection(1);
222  Print() << " domain is 2D in xz" << std::endl;
223  } else if (dom_lo.z == dom_hi.z) {
224  info.setHiddenDirection(2);
225  Print() << " domain is 2D in xy" << std::endl;
226  }
227 #endif
228 
229  // ****************************************************************************
230  // Solve nodal masked Poisson problem with MLMG
231  // TODO: different solver for terrain?
232  // ****************************************************************************
233  const Real reltol = solverChoice.poisson_reltol;
234  const Real abstol = solverChoice.poisson_abstol;
235 
236  Real sigma = 1.0;
237  Vector<EBFArrayBoxFactory const*> factory_vec = { &EBFactory(lev) };
238  MLNodeLaplacian mlpoisson(geom_tmp, ba_tmp, dm_tmp, info, factory_vec, sigma);
239 
240  mlpoisson.setDomainBC(bc3d_lo, bc3d_hi);
241 
242  if (lev > 0) {
243  mlpoisson.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
244  }
245 
246  mlpoisson.setLevelBC(0, nullptr);
247 
248  mlpoisson.setOversetMask(0, mask);
249 
250  // Solve
251  MLMG mlmg(mlpoisson);
252  int max_iter = 100;
253  mlmg.setMaxIter(max_iter);
254 
255  mlmg.setVerbose(mg_verbose);
256  mlmg.setBottomVerbose(0);
257 
258  mlmg.solve(GetVecOfPtrs(phi),
259  GetVecOfConstPtrs(rhs),
260  reltol, abstol);
261 
262  // Now overwrite with periodic fill outside domain and fine-fine fill inside
263  phi[0].FillBoundary(geom[lev].periodicity());
264 
265  // ****************************************************************************
266  // Compute grad(phi) to get distances
267  // - Note that phi is nodal and walldist is cell-centered
268  // - TODO: include terrain metrics for dphi/dz
269  // ****************************************************************************
270  for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
271  const Box& bx = mfi.validbox();
272 
273  const auto invCellSize = geomdata.InvCellSizeArray();
274 
275  auto const& phi_arr = phi[0].const_array(mfi);
276  auto dist_arr = walldist[lev]->array(mfi);
277 
278  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
279  Real dpdx{0}, dpdy{0}, dpdz{0};
280 
281  // dphi/dx
282  if (dom_lo.x != dom_hi.x) {
283  dpdx = 0.25 * invCellSize[0] * (
284  (phi_arr(i+1, j , k ) - phi_arr(i, j , k ))
285  + (phi_arr(i+1, j , k+1) - phi_arr(i, j , k+1))
286  + (phi_arr(i+1, j+1, k ) - phi_arr(i, j+1, k ))
287  + (phi_arr(i+1, j+1, k+1) - phi_arr(i, j+1, k+1)) );
288  }
289 
290  // dphi/dy
291  if (dom_lo.y != dom_hi.y) {
292  dpdy = 0.25 * invCellSize[1] * (
293  (phi_arr(i , j+1, k ) - phi_arr(i , j, k ))
294  + (phi_arr(i , j+1, k+1) - phi_arr(i , j, k+1))
295  + (phi_arr(i+1, j+1, k ) - phi_arr(i+1, j, k ))
296  + (phi_arr(i+1, j+1, k+1) - phi_arr(i+1, j, k+1)) );
297  }
298 
299  // dphi/dz
300  if (dom_lo.z != dom_hi.z) {
301  dpdz = 0.25 * invCellSize[2] * (
302  (phi_arr(i , j , k+1) - phi_arr(i , j , k))
303  + (phi_arr(i , j+1, k+1) - phi_arr(i , j+1, k))
304  + (phi_arr(i+1, j , k+1) - phi_arr(i+1, j , k))
305  + (phi_arr(i+1, j+1, k+1) - phi_arr(i+1, j+1, k)) );
306  }
307 
308  Real dp_dot_dp = dpdx*dpdx + dpdy*dpdy + dpdz*dpdz;
309  Real phi_avg = 0.125 * (
310  phi_arr(i , j , k ) + phi_arr(i , j , k+1) + phi_arr(i , j+1, k ) + phi_arr(i , j+1, k+1)
311  + phi_arr(i+1, j , k ) + phi_arr(i+1, j , k+1) + phi_arr(i+1, j+1, k ) + phi_arr(i+1, j+1, k+1) );
312  dist_arr(i, j, k) = -std::sqrt(dp_dot_dp) + std::sqrt(dp_dot_dp + 2*phi_avg);
313 
314  // DEBUG: output phi instead
315  //dist_arr(i, j, k) = phi_arr(i, j, k);
316  });
317  }
318 }
if(l_use_mynn &&start_comp<=RhoKE_comp &&end_comp >=RhoKE_comp)
Definition: ERF_AddQKESources.H:2
static int mg_verbose
Definition: ERF.H:1196
amrex::Real poisson_reltol
Definition: ERF_DataStruct.H:946
amrex::Real poisson_abstol
Definition: ERF_DataStruct.H:945
Here is the call graph for this function:

◆ post_timestep()

void ERF::post_timestep ( int  nstep,
amrex::Real  time,
amrex::Real  dt_lev 
)
662 {
663  BL_PROFILE("ERF::post_timestep()");
664 
665 #ifdef ERF_USE_PARTICLES
666  particleData.Redistribute();
667 #endif
668 
669  if (solverChoice.coupling_type == CouplingType::TwoWay)
670  {
671  int ncomp = vars_new[0][Vars::cons].nComp();
672  for (int lev = finest_level-1; lev >= 0; lev--)
673  {
674  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
675  // m is the map scale factor at cell centers
676  // Here we pre-divide (rho S) by m^2 before refluxing
677  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
678  const Box& bx = mfi.tilebox();
679  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
680  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
681  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
682  if (SolverChoice::mesh_type == MeshType::ConstantDz) {
683  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
684  {
685  cons_arr(i,j,k,n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
686  });
687  } else {
688  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
689  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
690  {
691  cons_arr(i,j,k,n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
692  });
693  }
694  } // mfi
695 
696  // This call refluxes all "slow" cell-centered variables
697  // (i.e. not density or (rho theta) or velocities) from the lev/lev+1 interface onto lev
698  getAdvFluxReg(lev+1)->Reflux(vars_new[lev][Vars::cons], 2, 2, ncomp-2);
699 
700  // Here we multiply (rho S) by m^2 after refluxing
701  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
702  const Box& bx = mfi.tilebox();
703  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
704  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
705  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
706  if (SolverChoice::mesh_type == MeshType::ConstantDz) {
707  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
708  {
709  cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
710  });
711  } else {
712  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
713  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
714  {
715  cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
716  });
717  }
718  } // mfi
719 
720  // We need to do this before anything else because refluxing changes the
721  // values of coarse cells underneath fine grids with the assumption they'll
722  // be over-written by averaging down
723  int src_comp;
724  if (solverChoice.anelastic[lev]) {
725  src_comp = 1;
726  } else {
727  src_comp = 0;
728  }
729  int num_comp = ncomp - src_comp;
730  AverageDownTo(lev,src_comp,num_comp);
731  }
732  }
733 
734  if (is_it_time_for_action(nstep, time, dt_lev0, sum_interval, sum_per)) {
737  sum_energy_quantities(time);
738  }
739 
740  if (solverChoice.pert_type == PerturbationType::Source ||
741  solverChoice.pert_type == PerturbationType::Direct ||
742  solverChoice.pert_type == PerturbationType::CPM) {
743  if (is_it_time_for_action(nstep, time, dt_lev0, pert_interval, -1.)) {
744  turbPert.debug(time);
745  }
746  }
747 
748  if (profile_int > 0 && (nstep+1) % profile_int == 0) {
749  if (destag_profiles) {
750  // all variables cell-centered
751  write_1D_profiles(time);
752  } else {
753  // some variables staggered
755  }
756  }
757 
758  if (solverChoice.rad_type != RadiationType::None)
759  {
760  if ( rad_datalog_int > 0 &&
761  (((nstep+1) % rad_datalog_int == 0) || (nstep==0)) ) {
762  if (rad[0]->hasDatalog()) {
763  rad[0]->WriteDataLog(time+start_time);
764  }
765  }
766  }
767 
768  if (output_1d_column) {
769 #ifdef ERF_USE_NETCDF
770  if (is_it_time_for_action(nstep, time, dt_lev0, column_interval, column_per))
771  {
772  int lev_column = 0;
773  for (int lev = finest_level; lev >= 0; lev--)
774  {
775  Real dx_lev = geom[lev].CellSize(0);
776  Real dy_lev = geom[lev].CellSize(1);
777  int i_lev = static_cast<int>(std::floor(column_loc_x / dx_lev));
778  int j_lev = static_cast<int>(std::floor(column_loc_y / dy_lev));
779  if (grids[lev].contains(IntVect(i_lev,j_lev,0))) lev_column = lev;
780  }
781  writeToNCColumnFile(lev_column, column_file_name, column_loc_x, column_loc_y, time);
782  }
783 #else
784  Abort("To output 1D column files ERF must be compiled with NetCDF");
785 #endif
786  }
787 
789  {
792  {
793  bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
794  m_w2d->write_planes(istep[0], time, vars_new, is_moist);
795  }
796  }
797 
798  // Write plane/line sampler data
800  line_sampler->get_sample_data(geom, vars_new);
801  line_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
802  }
804  plane_sampler->get_sample_data(geom, vars_new);
805  plane_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
806  }
807 
808  // Moving terrain
809  if ( solverChoice.terrain_type == TerrainType::MovingFittedMesh )
810  {
811  for (int lev = finest_level; lev >= 0; lev--)
812  {
813  // Copy z_phs_nd and detJ_cc at end of timestep
814  MultiFab::Copy(*z_phys_nd[lev], *z_phys_nd_new[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect());
815  MultiFab::Copy( *detJ_cc[lev], *detJ_cc_new[lev], 0, 0, 1, detJ_cc[lev]->nGrowVect());
816  MultiFab::Copy(base_state[lev],base_state_new[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
817 
818  make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
819  }
820  }
821 
822  bool is_hurricane_tracker_io=false;
823  ParmParse pp("erf");
824  pp.query("is_hurricane_tracker_io", is_hurricane_tracker_io);
825 
826  if (is_hurricane_tracker_io) {
827  if(nstep == 0 or (nstep+1)%m_plot3d_int_1 == 0){
828  std::string filename = MakeVTKFilename(nstep);
829  Real velmag_threshold = 1e10;
830  pp.query("hurr_track_io_velmag_greater_than", velmag_threshold);
831  if(velmag_threshold==1e10) {
832  Abort("As hurricane tracking IO is active using erf.is_hurricane_tracker_io = true"
833  " there needs to be an input erf.hurr_track_io_velmag_greater_than which specifies the"
834  " magnitude of velocity above which cells will be tagged for refinement.");
835  }
836  int levc=finest_level;
837  MultiFab& U_new = vars_new[levc][Vars::xvel];
838  MultiFab& V_new = vars_new[levc][Vars::yvel];
839  MultiFab& W_new = vars_new[levc][Vars::zvel];
840 
841  HurricaneTracker(levc, U_new, V_new, W_new, velmag_threshold, true);
842  if (ParallelDescriptor::IOProcessor()) {
844  }
845  }
846  }
847 
848  if(solverChoice.io_hurricane_eye_tracker and (nstep == 0 or (nstep+1)%m_plot3d_int_1 == 0)) {
849  int levc=finest_level;
850 
851  HurricaneEyeTracker(geom[levc],
852  vars_new[levc],
860 
861  MultiFab& U_new = vars_new[levc][Vars::xvel];
862  MultiFab& V_new = vars_new[levc][Vars::yvel];
863  MultiFab& W_new = vars_new[levc][Vars::zvel];
864 
865  MultiFab mf_cc_vel(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(0,0,0));
866  average_face_to_cellcenter(mf_cc_vel,0,{AMREX_D_DECL(&U_new,&V_new,&W_new)},0);
867 
868  HurricaneMaxVelTracker(geom[levc],
869  mf_cc_vel,
870  t_new[0],
873 
874  std::string filename_tracker = MakeVTKFilename_TrackerCircle(nstep);
875  std::string filename_xy = MakeVTKFilename_EyeTracker_xy(nstep);
876  std::string filename_latlon = MakeFilename_EyeTracker_latlon(nstep);
877  std::string filename_maxvel = MakeFilename_EyeTracker_maxvel(nstep);
878  if (ParallelDescriptor::IOProcessor()) {
879  WriteVTKPolyline(filename_tracker, hurricane_tracker_circle);
881  WriteLinePlot(filename_latlon, hurricane_eye_track_latlon);
882  WriteLinePlot(filename_maxvel, hurricane_maxvel_vs_time);
883  }
884  }
885 
886 } // post_timestep
void HurricaneMaxVelTracker(const amrex::Geometry &geom, const amrex::MultiFab &mf_cc_vel, const amrex::Real &time, const amrex::Vector< std::array< amrex::Real, 2 >> &hurricane_eye_track_xy, amrex::Vector< std::array< amrex::Real, 2 >> &hurricane_maxvel_vs_time)
Definition: ERF_HurricaneDiagnostics.H:281
void HurricaneEyeTracker(const amrex::Geometry &geom, const amrex::Vector< amrex::MultiFab > &S_data, MoistureType moisture_type, const amrex::Vector< amrex::MultiFab > *forecast_state_at_lev, const amrex::Real &hurricane_eye_latitude, const amrex::Real &hurricane_eye_longitude, amrex::Vector< std::array< amrex::Real, 2 >> &hurricane_eye_track_xy, amrex::Vector< std::array< amrex::Real, 2 >> &hurricane_eye_track_latlon, amrex::Vector< std::array< amrex::Real, 2 >> &hurricane_tracker_circle)
Definition: ERF_HurricaneDiagnostics.H:255
void make_zcc(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &z_phys_cc)
Definition: ERF_TerrainMetrics.cpp:624
std::string MakeFilename_EyeTracker_maxvel(int nstep)
Definition: ERF_Write1DProfiles.cpp:650
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_eye_track_xy
Definition: ERF.H:154
static amrex::Real column_loc_y
Definition: ERF.H:1257
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_interp
Definition: ERF.H:161
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_tracker_circle
Definition: ERF.H:157
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_maxvel_vs_time
Definition: ERF.H:156
static std::string column_file_name
Definition: ERF.H:1258
AMREX_FORCE_INLINE amrex::YAFluxRegister * getAdvFluxReg(int lev)
Definition: ERF.H:1407
static amrex::Real bndry_output_planes_per
Definition: ERF.H:1263
std::string MakeVTKFilename(int nstep)
Definition: ERF_Write1DProfiles.cpp:574
static amrex::Real column_per
Definition: ERF.H:1255
static amrex::Real column_loc_x
Definition: ERF.H:1256
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_eye_track_latlon
Definition: ERF.H:155
std::string MakeVTKFilename_TrackerCircle(int nstep)
Definition: ERF_Write1DProfiles.cpp:593
std::string MakeVTKFilename_EyeTracker_xy(int nstep)
Definition: ERF_Write1DProfiles.cpp:612
static int bndry_output_planes_interval
Definition: ERF.H:1262
void WriteLinePlot(const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
Definition: ERF_Write1DProfiles.cpp:716
static int output_1d_column
Definition: ERF.H:1253
void WriteVTKPolyline(const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
Definition: ERF_Write1DProfiles.cpp:669
std::string MakeFilename_EyeTracker_latlon(int nstep)
Definition: ERF_Write1DProfiles.cpp:631
static int column_interval
Definition: ERF.H:1254
amrex::Real hurricane_eye_latitude
Definition: ERF_DataStruct.H:1077
amrex::Real hurricane_eye_longitude
Definition: ERF_DataStruct.H:1077
bool io_hurricane_eye_tracker
Definition: ERF_DataStruct.H:1076
Here is the call graph for this function:

◆ post_update()

void ERF::post_update ( amrex::MultiFab &  state_mf,
amrex::Real  time,
const amrex::Geometry &  geom 
)
private

◆ print_banner()

void ERF::print_banner ( MPI_Comm  comm,
std::ostream &  out 
)
static
65  : " << msg << std::endl;
66 }
67 
68 void ERF::print_banner (MPI_Comm comm, std::ostream& out)
69 {
70 #ifdef AMREX_USE_MPI
71  int irank = 0;
72  int num_ranks = 1;
73  MPI_Comm_size(comm, &num_ranks);
74  MPI_Comm_rank(comm, &irank);
75 
76  // Only root process does the printing
77  if (irank != 0) return;
78 #else
79  amrex::ignore_unused(comm);
80 #endif
81 
82  auto etime = std::chrono::system_clock::now();
83  auto etimet = std::chrono::system_clock::to_time_t(etime);
84 #ifndef _WIN32
85  char time_buf[64];
86  ctime_r(&etimet, time_buf);
87  const std::string tstamp(time_buf);
88 #else
89  char* time_buf = new char[64];
90  ctime_s(time_buf, 64, &etimet);
91  const std::string tstamp(time_buf);
92 #endif
93 
94  const char* githash1 = amrex::buildInfoGetGitHash(1);
95  const char* githash2 = amrex::buildInfoGetGitHash(2);
96 
97  // clang-format off
98  out << dbl_line
99  << " ERF (https://github.com/erf-model/ERF)"
100  << std::endl << std::endl
101  << " ERF Git SHA :: " << githash1 << std::endl
102  << " AMReX Git SHA :: " << githash2 << std::endl
103  << " AMReX version :: " << amrex::Version() << std::endl << std::endl
104  << " Exec. time :: " << tstamp
105  << " Build time :: " << amrex::buildInfoGetBuildDate() << std::endl
106  << " C++ compiler :: " << amrex::buildInfoGetComp()
107  << " " << amrex::buildInfoGetCompVersion() << std::endl << std::endl
108  << " MPI :: "
109 #ifdef AMREX_USE_MPI
110  << "ON (Num. ranks = " << num_ranks << ")" << std::endl
111 #else
112  << "OFF " << std::endl
113 #endif
114  << " GPU :: "
115 #ifdef AMREX_USE_GPU
116  << "ON "
117 #if defined(AMREX_USE_CUDA)
118  << "(Backend: CUDA)"
119 #elif defined(AMREX_USE_HIP)
120  << "(Backend: HIP)"
121 #elif defined(AMREX_USE_SYCL)
122  << "(Backend: SYCL)"
123 #endif
124  << std::endl
125 #else
126  << "OFF" << std::endl
127 #endif
128  << " OpenMP :: "
129 #ifdef AMREX_USE_OMP
130  << "ON (Num. threads = " << omp_get_max_threads() << ")" << std::endl
131 #else
132  << "OFF" << std::endl
133 #endif
134  << std::endl;
135 
ERF()
Definition: ERF.cpp:128
const char * buildInfoGetBuildDate()
const char * buildInfoGetComp()
const char * buildInfoGetCompVersion()

Referenced by main().

Here is the caller graph for this function:

◆ print_error()

void ERF::print_error ( MPI_Comm  comm,
const std::string &  msg 
)
static
43  :
44  input_file : Input file with simulation settings
45 
46 Optional:
47  param=value : Overrides for parameters during runtime
48 )doc" << std::endl;
49 }
50 
51 void ERF::print_error (MPI_Comm comm, const std::string& msg)
52 {
53 #ifdef AMREX_USE_MPI
54  int irank = 0;
55  int num_ranks = 1;
56  MPI_Comm_size(comm, &num_ranks);
57  MPI_Comm_rank(comm, &irank);
58 
amrex::Real value
Definition: ERF_HurricaneDiagnostics.H:20

Referenced by main().

Here is the caller graph for this function:

◆ print_summary()

static void ERF::print_summary ( std::ostream &  )
static

◆ print_tpls()

void ERF::print_tpls ( std::ostream &  out)
static
140  ://github.com/erf-model/ERF/blob/development/LICENSE for details. "
141  << dash_line << std::endl;
142  // clang-format on
143 }
144 
145 void ERF::print_tpls (std::ostream& out)
146 {
147  amrex::Vector<std::string> tpls;
148 
149 #ifdef ERF_USE_NETCDF
150  tpls.push_back(std::string("NetCDF ") + NC_VERSION);
151 #endif
152 #ifdef AMREX_USE_SUNDIALS
153  tpls.push_back(std::string("SUNDIALS ") + SUNDIALS_VERSION);
154 #endif
155 
156  if (!tpls.empty()) {
157  out << " Enabled third-party libraries: ";
158  for (const auto& val : tpls) {
struct @19 out
static void print_tpls(std::ostream &)
Definition: ERF_ConsoleIO.cpp:137

◆ print_usage()

void ERF::print_usage ( MPI_Comm  comm,
std::ostream &  out 
)
static
27 {
28 #ifdef AMREX_USE_MPI
29  int irank = 0;
30  int num_ranks = 1;
31  MPI_Comm_size(comm, &num_ranks);
32  MPI_Comm_rank(comm, &irank);
33 
34  // Only root process does the printing
35  if (irank != 0) return;
36 #else
37  amrex::ignore_unused(comm);
38 #endif
39 
40  out << R"doc(Usage:
41  ERF3d.*.ex <input_file> [param=value] [param=value] ...

Referenced by main().

Here is the caller graph for this function:

◆ project_momenta()

void ERF::project_momenta ( int  lev,
amrex::Real  dt,
amrex::Vector< amrex::MultiFab > &  vars 
)

Project the single-level momenta to enforce the anelastic constraint Note that the level may or may not be level 0.

45 {
46  BL_PROFILE("ERF::project_momenta()");
47 
48  // Make sure the solver only sees the levels over which we are solving
49  Vector<BoxArray> ba_tmp; ba_tmp.push_back(mom_mf[Vars::cons].boxArray());
50  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(mom_mf[Vars::cons].DistributionMap());
51  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
52 
53  Box domain = geom[lev].Domain();
54 
55  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
56 
57  Vector<MultiFab> rhs;
58  Vector<MultiFab> phi;
59 
60  if (solverChoice.terrain_type == TerrainType::EB)
61  {
62  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
63  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1, MFInfo(), EBFactory(lev));
64  } else {
65  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
66  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
67  }
68 
69  MultiFab rhs_lev(rhs[0], make_alias, 0, 1);
70  MultiFab phi_lev(phi[0], make_alias, 0, 1);
71 
72  auto dxInv = geom[lev].InvCellSizeArray();
73 
74  // Inflow on an x-face -- note only the normal velocity is used in the projection
75  if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow") {
77  IntVect{1,0,0},t_new[lev],BCVars::xvel_bc,false);
78  }
79 
80  // Inflow on a y-face -- note only the normal velocity is used in the projection
81  if (domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
83  IntVect{0,1,0},t_new[lev],BCVars::yvel_bc,false);
84  }
85 
86  if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow" ||
87  domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
88  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
89  vars_new[lev][Vars::yvel], IntVect{0},
90  vars_new[lev][Vars::zvel], IntVect{0},
91  vars_new[lev][Vars::cons],
92  mom_mf[IntVars::xmom],
93  mom_mf[IntVars::ymom],
94  mom_mf[IntVars::zmom],
95  Geom(lev).Domain(),
97  }
98 
99  // If !fixed_density, we must convert (rho u) which came in
100  // to (rho0 u) which is what we will project
102  ConvertForProjection(mom_mf[Vars::cons], r_hse,
103  mom_mf[IntVars::xmom],
104  mom_mf[IntVars::ymom],
105  mom_mf[IntVars::zmom],
106  Geom(lev).Domain(),
108  }
109 
110  //
111  // ****************************************************************************
112  // Now convert the rho0w MultiFab to hold Omega rather than rhow
113  // ****************************************************************************
114  //
115  if (solverChoice.mesh_type == MeshType::VariableDz)
116  {
117  for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
118  {
119  const Array4<Real const>& rho0u_arr = mom_mf[IntVars::xmom].const_array(mfi);
120  const Array4<Real const>& rho0v_arr = mom_mf[IntVars::ymom].const_array(mfi);
121  const Array4<Real >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
122 
123  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
124  const Array4<Real const>& mf_u = mapfac[lev][MapFacType::u_x]->const_array(mfi);
125  const Array4<Real const>& mf_v = mapfac[lev][MapFacType::v_y]->const_array(mfi);
126 
127  //
128  // Define Omega from (rho0 W) but store it in the same array
129  //
130  Box tbz = mfi.nodaltilebox(2);
131  ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
132  if (k == 0) {
133  rho0w_arr(i,j,k) = Real(0.0);
134  } else {
135  Real rho0w = rho0w_arr(i,j,k);
136  rho0w_arr(i,j,k) = OmegaFromW(i,j,k,rho0w,
137  rho0u_arr,rho0v_arr,
138  mf_u,mf_v,z_nd,dxInv);
139  }
140  });
141  } // mfi
142  }
143 
144  // ****************************************************************************
145  // Allocate fluxes
146  // ****************************************************************************
147  Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
148  fluxes.resize(1);
149  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
150  if (solverChoice.terrain_type == TerrainType::EB) {
151  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
152  } else {
153  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
154  }
155  }
156 
157  Array<MultiFab const*, AMREX_SPACEDIM> rho0_u_const;
158  rho0_u_const[0] = &mom_mf[IntVars::xmom];
159  rho0_u_const[1] = &mom_mf[IntVars::ymom];
160  rho0_u_const[2] = &mom_mf[IntVars::zmom];
161 
162  // ****************************************************************************
163  // Initialize phi to 0
164  // (It is essential that we do this in order to fill the corners; these are never
165  // used but the Saxpy requires the values to be initialized.)
166  // ****************************************************************************
167  phi_lev.setVal(0.0);
168 
169  // ****************************************************************************
170  // Break into subdomains
171  // ****************************************************************************
172 
173  std::map<int,int> index_map;
174 
175  BoxArray ba(grids[lev]);
176 
177  Vector<MultiFab> rhs_sub; rhs_sub.resize(1);
178  Vector<MultiFab> phi_sub; phi_sub.resize(1);
179  Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes_sub; fluxes_sub.resize(1);
180 
181  MultiFab ax_sub, ay_sub, az_sub, dJ_sub, znd_sub;
182 
183  for (int isub = 0; isub < subdomains[lev].size(); ++isub)
184  {
185  BoxList bl_sub;
186  Vector<int> dm_sub;
187 
188  for (int j = 0; j < ba.size(); j++)
189  {
190  if (subdomains[lev][isub].intersects(ba[j]))
191  {
192  // amrex::Print() <<" INTERSECTS I " << isub << " " << j << " " << grids[lev][j] << std::endl;
193  //
194  // Note that bl_sub.size() is effectively a counter which is
195  // incremented above
196  //
197  // if (ParallelDescriptor::MyProc() == j) {
198  // }
199  index_map[bl_sub.size()] = j;
200 
201  // amrex::Print() <<" PUSHING BACK " << j << " " << index_map[bl_sub.size()] << std::endl;
202  bl_sub.push_back(grids[lev][j]);
203  dm_sub.push_back(dmap[lev][j]);
204  } // intersects
205 
206  } // loop over ba (j)
207 
208  BoxArray ba_sub(bl_sub);
209 
210  // Define MultiFabs that hold only the data in this particular subdomain
211  rhs_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, rhs_lev.nGrowVect(), MFInfo{}.SetAlloc(false));
212  phi_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, phi_lev.nGrowVect(), MFInfo{}.SetAlloc(false));
213 
214  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
215  fluxes_sub[0][idim].define(convert(ba_sub, IntVect::TheDimensionVector(idim)), DistributionMapping(dm_sub), 1,
216  IntVect::TheZeroVector(), MFInfo{}.SetAlloc(false));
217  }
218 
219  // Link the new MultiFabs to the FABs in the original MultiFabs (no copy required)
220  for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
221  int orig_index = index_map[mfi.index()];
222  // amrex::Print() << " INDEX " << orig_index << " TO " << mfi.index() << std::endl;
223  rhs_sub[0].setFab(mfi, FArrayBox(rhs_lev[orig_index], amrex::make_alias, 0, 1));
224  phi_sub[0].setFab(mfi, FArrayBox(phi_lev[orig_index], amrex::make_alias, 0, 1));
225  fluxes_sub[0][0].setFab(mfi,FArrayBox(fluxes[0][0][orig_index], amrex::make_alias, 0, 1));
226  fluxes_sub[0][1].setFab(mfi,FArrayBox(fluxes[0][1][orig_index], amrex::make_alias, 0, 1));
227  fluxes_sub[0][2].setFab(mfi,FArrayBox(fluxes[0][2][orig_index], amrex::make_alias, 0, 1));
228  }
229 
230  if (solverChoice.mesh_type != MeshType::ConstantDz) {
231  ax_sub.define(convert(ba_sub,IntVect(1,0,0)), DistributionMapping(dm_sub), 1,
232  ax[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
233  ay_sub.define(convert(ba_sub,IntVect(0,1,0)), DistributionMapping(dm_sub), 1,
234  ay[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
235  az_sub.define(convert(ba_sub,IntVect(0,0,1)), DistributionMapping(dm_sub), 1,
236  az[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
237  znd_sub.define(convert(ba_sub,IntVect(1,1,1)), DistributionMapping(dm_sub), 1,
238  z_phys_nd[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
239  dJ_sub.define(ba_sub, DistributionMapping(dm_sub), 1,
240  detJ_cc[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
241 
242  for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
243  int orig_index = index_map[mfi.index()];
244  ax_sub.setFab(mfi, FArrayBox((*ax[lev])[orig_index], amrex::make_alias, 0, 1));
245  ay_sub.setFab(mfi, FArrayBox((*ay[lev])[orig_index], amrex::make_alias, 0, 1));
246  az_sub.setFab(mfi, FArrayBox((*az[lev])[orig_index], amrex::make_alias, 0, 1));
247  znd_sub.setFab(mfi, FArrayBox((*z_phys_nd[lev])[orig_index], amrex::make_alias, 0, 1));
248  dJ_sub.setFab(mfi, FArrayBox((*detJ_cc[lev])[orig_index], amrex::make_alias, 0, 1));
249  }
250  }
251 
252  // ****************************************************************************
253  // Compute divergence which will form RHS
254  // Note that we replace "rho0w" with the contravariant momentum, Omega
255  // ****************************************************************************
256 
257  compute_divergence(lev, rhs_sub[0], rho0_u_const, geom_tmp[0]);
258 
259  Real rhsnorm;
260 
261  // Max norm over the entire MultiFab
262  rhsnorm = rhs_lev.norm0();
263 
264  if (mg_verbose > 0) {
265  Real sum = volWgtSumMF(lev,rhs_lev,0,false);
266  ParallelDescriptor::ReduceRealSum(sum);
267  Print() << "Max/L2 norm of divergence before solve in subdomain " << isub << " at level " << lev << " : " << rhsnorm << " " <<
268  rhs_lev.norm2() << " and volume-weighted sum " << sum << std::endl;
269  }
270 
271  if (lev == 0 && solverChoice.use_real_bcs)
272  {
273  // We always use VariableDz if use_real_bcs is true
274  AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::VariableDz);
275 
276  // Note that we always impose the projections one level at a time so this will always be a vector of length 1
277  Array<MultiFab*, AMREX_SPACEDIM> rho0_u_vec =
278  {&mom_mf[IntVars::xmom], &mom_mf[IntVars::ymom], &mom_mf[IntVars::zmom]};
279  Array<MultiFab*, AMREX_SPACEDIM> area_vec = {ax[lev].get(), ay[lev].get(), az[lev].get()};
280  //
281  // Modify ax,ay,ax to include the map factors as used in the divergence calculation
282  // We do this here so that it is seen in the call to enforceInOutSolvability
283  //
284  for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
285  {
286  Box xbx = mfi.nodaltilebox(0);
287  Box ybx = mfi.nodaltilebox(1);
288  Box zbx = mfi.nodaltilebox(2);
289  const Array4<Real >& ax_ar = ax[lev]->array(mfi);
290  const Array4<Real >& ay_ar = ay[lev]->array(mfi);
291  const Array4<Real >& az_ar = az[lev]->array(mfi);
292  const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
293  const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
294  const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
295  const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
296  ParallelFor(xbx,ybx,zbx,
297  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
298  {
299  ax_ar(i,j,k) /= mf_uy(i,j,0);
300  },
301  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
302  {
303  ay_ar(i,j,k) /= mf_vx(i,j,0);
304  },
305  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
306  {
307  az_ar(i,j,k) /= (mf_mx(i,j,0)*mf_my(i,j,0));
308  });
309  } // mfi
310 
311  if (mg_verbose > 0) {
312  Print() << "Calling enforceInOutSolvability" << std::endl;
313  }
314  enforceInOutSolvability(lev, rho0_u_vec, area_vec, geom[lev]);
315 
316  //
317  // Return ax,ay,ax to their original definition
318  //
319  for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
320  {
321  Box xbx = mfi.nodaltilebox(0);
322  Box ybx = mfi.nodaltilebox(1);
323  Box zbx = mfi.nodaltilebox(2);
324  const Array4<Real >& ax_ar = ax[lev]->array(mfi);
325  const Array4<Real >& ay_ar = ay[lev]->array(mfi);
326  const Array4<Real >& az_ar = az[lev]->array(mfi);
327  const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
328  const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
329  const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
330  const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
331  ParallelFor(xbx,ybx,zbx,
332  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
333  {
334  ax_ar(i,j,k) *= mf_uy(i,j,0);
335  },
336  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
337  {
338  ay_ar(i,j,k) *= mf_vx(i,j,0);
339  },
340  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
341  {
342  az_ar(i,j,k) *= (mf_mx(i,j,0)*mf_my(i,j,0));
343  });
344  } // mfi
345 
346  compute_divergence(lev, rhs_lev, rho0_u_const, geom_tmp[0]);
347 
348  // Re-define max norm over the entire MultiFab
349  rhsnorm = rhs_lev.norm0();
350 
351  if (mg_verbose > 0)
352  {
353  Real sum = volWgtSumMF(lev,rhs_lev,0,false);
354  ParallelDescriptor::ReduceRealSum(sum);
355  Print() << "Max/L2 norm of divergence before solve at level " << lev << " : " << rhsnorm << " " <<
356  rhs_lev.norm2() << " and volume-weighted sum " << sum << std::endl;
357  }
358  } // lev 0 && use_real_bcs
359 
360  // *******************************************************************************************
361  // Enforce solvability if the problem is singular (i.e all sides Neumann or periodic)
362  // Note that solves at lev > 0 are always singular because we impose Neumann bc's on all sides
363  // *******************************************************************************************
364  bool is_singular = true;
365  if (lev == 0) {
366  if ( (domain_bc_type[0] == "Outflow" || domain_bc_type[0] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
367  if ( (domain_bc_type[1] == "Outflow" || domain_bc_type[1] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
368  if ( (domain_bc_type[3] == "Outflow" || domain_bc_type[3] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
369  if ( (domain_bc_type[4] == "Outflow" || domain_bc_type[4] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
370  if ( (domain_bc_type[5] == "Outflow" || domain_bc_type[5] == "Open") ) is_singular = false;
371  } else {
372  Box my_region(subdomains[lev][isub].minimalBox());
373  if ( (domain_bc_type[5] == "Outflow" || domain_bc_type[5] == "Open") && (my_region.bigEnd(2) == domain.bigEnd(2)) ) is_singular = false;
374  }
375 
376  if (is_singular)
377  {
378  Real sum = volWgtSumMF(lev,rhs_sub[0],0,false);
379  ParallelDescriptor::ReduceRealSum(sum);
380 
381  Real vol;
382  if (solverChoice.mesh_type == MeshType::ConstantDz) {
383  vol = rhs_sub[0].boxArray().numPts();
384  } else {
385  vol = dJ_sub.sum() / (dxInv[0] * dxInv[1] * dxInv[2]);
386  }
387  sum /= vol;
388 
389  for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
390  {
391  rhs_sub[0][mfi.index()].template minus<RunOn::Device>(sum);
392  }
393  if (mg_verbose > 0) {
394  amrex::Print() << " Subtracting " << sum << " from rhs in subdomain " << isub << std::endl;
395 
396  sum = volWgtSumMF(lev,rhs_sub[0],0,false);
397  Print() << "Sum after subtraction " << sum << " in subdomain " << isub << std::endl;
398  }
399 
400  } // if is_singular
401 
402  rhsnorm = rhs_sub[0].norm0();
403 
404  // ****************************************************************************
405  // No need to build the solver if RHS == 0
406  // ****************************************************************************
407  if (rhsnorm <= solverChoice.poisson_abstol) return;
408 
409  Real start_step = static_cast<Real>(ParallelDescriptor::second());
410 
411  if (mg_verbose > 0) {
412  amrex::Print() << " Solving in subdomain " << isub << " of " << subdomains[lev].size() << " bins at level " << lev << std::endl;
413  }
414 
415  if (solverChoice.mesh_type == MeshType::VariableDz) {
416  //
417  // Modify ax,ay,ax to include the map factors as used in the divergence calculation
418  // We do this here to set the coefficients used in the stencil -- the extra factor
419  // of the mapfac comes from the gradient
420  //
421  for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
422  {
423  Box xbx = mfi.nodaltilebox(0);
424  Box ybx = mfi.nodaltilebox(1);
425  Box zbx = mfi.nodaltilebox(2);
426  const Array4<Real >& ax_ar = ax_sub.array(mfi);
427  const Array4<Real >& ay_ar = ay_sub.array(mfi);
428  const Array4<Real >& az_ar = az_sub.array(mfi);
429  const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
430  const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
431  const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
432  const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
433  const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi); const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
434  ParallelFor(xbx,ybx,zbx,
435  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
436  {
437  ax_ar(i,j,k) *= (mf_ux(i,j,0) / mf_uy(i,j,0));
438  },
439  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
440  {
441  ay_ar(i,j,k) *= (mf_vy(i,j,0) / mf_vx(i,j,0));
442  },
443  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
444  {
445  az_ar(i,j,k) /= (mf_mx(i,j,0)*mf_my(i,j,0));
446  });
447  } // mfi
448  }
449 
450  if (lev > 0) {
451  amrex::Print() << "RHSSUB BA " << rhs_sub[0].boxArray() << std::endl;
452  }
453 
454  // ****************************************************************************
455  // EB
456  // ****************************************************************************
457  if (solverChoice.terrain_type == TerrainType::EB) {
458  solve_with_EB_mlmg(lev, rhs_sub, phi_sub, fluxes_sub);
459  } else {
460 
461  // ****************************************************************************
462  // No terrain or grid stretching
463  // ****************************************************************************
464  if (solverChoice.mesh_type == MeshType::ConstantDz) {
465 #ifdef ERF_USE_FFT
466  if (use_fft) {
467  Box my_region(subdomains[lev][isub].minimalBox());
468  bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
469  if (boxes_make_rectangle) {
470  solve_with_fft(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
471  } else {
472  amrex::Warning("FFT won't work unless the union of boxes is rectangular: defaulting to MLMG");
473  solve_with_mlmg(lev, rhs_sub, phi_sub, fluxes_sub);
474  }
475  } else {
476  solve_with_mlmg(lev, rhs, phi, fluxes);
477  }
478 #else
479  if (use_fft) {
480  amrex::Warning("You set use_fft=true but didn't build with USE_FFT = TRUE; defaulting to MLMG");
481  }
482  solve_with_mlmg(lev, rhs_sub, phi_sub, fluxes_sub);
483 #endif
484  } // No terrain or grid stretching
485 
486  // ****************************************************************************
487  // Grid stretching (flat terrain)
488  // ****************************************************************************
489  else if (solverChoice.mesh_type == MeshType::StretchedDz) {
490 #ifndef ERF_USE_FFT
491  amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT solver");
492 #else
493  Box my_region(subdomains[lev][isub].minimalBox());
494  bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
495  if (!boxes_make_rectangle) {
496  amrex::Abort("FFT won't work unless the union of boxes is rectangular");
497  } else {
498  if (!use_fft) {
499  amrex::Warning("Using FFT even though you didn't set use_fft to true; it's the best choice");
500  }
501  solve_with_fft(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
502  }
503 #endif
504  } // grid stretching
505 
506  // ****************************************************************************
507  // General terrain
508  // ****************************************************************************
509  else if (solverChoice.mesh_type == MeshType::VariableDz) {
510 #ifdef ERF_USE_FFT
511  Box my_region(subdomains[lev][isub].minimalBox());
512  bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
513  if (!boxes_make_rectangle) {
514  amrex::Abort("FFT preconditioner for GMRES won't work unless the union of boxes is rectangular");
515  } else {
516  solve_with_gmres(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0], ax_sub, ay_sub, az_sub, dJ_sub, znd_sub);
517  }
518 #else
519  amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT preconditioner for GMRES");
520 #endif
521 
522  //
523  // Restore ax,ay,ax to their original definitions
524  //
525  for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
526  {
527  Box xbx = mfi.nodaltilebox(0);
528  Box ybx = mfi.nodaltilebox(1);
529  Box zbx = mfi.nodaltilebox(2);
530  const Array4<Real >& ax_ar = ax_sub.array(mfi);
531  const Array4<Real >& ay_ar = ay_sub.array(mfi);
532  const Array4<Real >& az_ar = az_sub.array(mfi);
533  const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
534  const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
535  const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
536  const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
537  const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
538  const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
539  ParallelFor(xbx,ybx,zbx,
540  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
541  {
542  ax_ar(i,j,k) *= (mf_uy(i,j,0) / mf_ux(i,j,0));
543  },
544  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
545  {
546  ay_ar(i,j,k) *= (mf_vx(i,j,0) / mf_vy(i,j,0));
547  },
548  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
549  {
550  az_ar(i,j,k) *= (mf_mx(i,j,0)*mf_my(i,j,0));
551  });
552  } // mfi
553 
554  } // MeshType::VariableDz
555 
556  } // not EB
557 
558  // ****************************************************************************
559  // Print time in solve
560  // ****************************************************************************
561  Real end_step = static_cast<Real>(ParallelDescriptor::second());
562  if (mg_verbose > 0) {
563  amrex::Print() << "Time in solve " << end_step - start_step << std::endl;
564  }
565  } // loop over subdomains (i)
566 
567  // ****************************************************************************
568  // Subtract dt grad(phi) from the momenta (rho0u, rho0v, Omega)
569  // ****************************************************************************
570  MultiFab::Add(mom_mf[IntVars::xmom],fluxes[0][0],0,0,1,0);
571  MultiFab::Add(mom_mf[IntVars::ymom],fluxes[0][1],0,0,1,0);
572  MultiFab::Add(mom_mf[IntVars::zmom],fluxes[0][2],0,0,1,0);
573 
574  // ****************************************************************************
575  // Define gradp from fluxes -- note that fluxes is dt * change in Gp
576  // (weighted by map factor!)
577  // ****************************************************************************
578  MultiFab::Saxpy(gradp[lev][GpVars::gpx],-1.0/l_dt,fluxes[0][0],0,0,1,0);
579  MultiFab::Saxpy(gradp[lev][GpVars::gpy],-1.0/l_dt,fluxes[0][1],0,0,1,0);
580  MultiFab::Saxpy(gradp[lev][GpVars::gpz],-1.0/l_dt,fluxes[0][2],0,0,1,0);
581 
582  gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
583  gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
584  gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
585 
586  //
587  // This call is only to verify the divergence after the solve
588  // It is important we do this before computing the rho0w_arr from Omega back to rho0w
589  //
590  // ****************************************************************************
591  // THIS IS SIMPLY VERIFYING THE DIVERGENCE AFTER THE SOLVE
592  // ****************************************************************************
593  //
594  if (mg_verbose > 0)
595  {
596  compute_divergence(lev, rhs_lev, rho0_u_const, geom_tmp[0]);
597 
598  Real sum = volWgtSumMF(lev,rhs_lev,0,false);
599 
600  if (mg_verbose > 0) {
601  Print() << "Max/L2 norm of divergence after solve at level " << lev << " : " << rhs_lev.norm0() << " " <<
602  rhs_lev.norm2() << " and volume-weighted sum " << sum << std::endl;
603  }
604 
605 #if 0
606  // FOR DEBUGGING ONLY
607  for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
608  {
609  const Array4<Real const>& rhs_arr = rhs_lev.const_array(mfi);
610  Box bx = mfi.validbox();
611  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
612  if (std::abs(rhs_arr(i,j,k)) > 1.e-10) {
613  amrex::AllPrint() << "RHS AFTER SOLVE AT " <<
614  IntVect(i,j,k) << " " << rhs_arr(i,j,k) << std::endl;
615  }
616  });
617  } // mfi
618 #endif
619 
620  } // mg_verbose
621 
622  //
623  // ****************************************************************************
624  // Now convert the rho0w MultiFab back to holding (rho0w) rather than Omega
625  // ****************************************************************************
626  //
627  if (solverChoice.mesh_type == MeshType::VariableDz)
628  {
629  for (MFIter mfi(mom_mf[Vars::cons],TilingIfNotGPU()); mfi.isValid(); ++mfi)
630  {
631  Box tbz = mfi.nodaltilebox(2);
632  const Array4<Real >& rho0u_arr = mom_mf[IntVars::xmom].array(mfi);
633  const Array4<Real >& rho0v_arr = mom_mf[IntVars::ymom].array(mfi);
634  const Array4<Real >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
635  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
636  const Array4<Real const>& mf_u = mapfac[lev][MapFacType::u_x]->const_array(mfi);
637  const Array4<Real const>& mf_v = mapfac[lev][MapFacType::v_y]->const_array(mfi);
638  ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
639  Real omega = rho0w_arr(i,j,k);
640  rho0w_arr(i,j,k) = WFromOmega(i,j,k,omega,
641  rho0u_arr,rho0v_arr,
642  mf_u,mf_v,z_nd,dxInv);
643  });
644  } // mfi
645  }
646 
647  // If !fixed_density, we must convert (rho0 u) back
648  // to (rho0 u) which is what we will pass back out
650  ConvertForProjection(r_hse, mom_mf[Vars::cons],
651  mom_mf[IntVars::xmom],
652  mom_mf[IntVars::ymom],
653  mom_mf[IntVars::zmom],
654  Geom(lev).Domain(),
656  }
657 
658  // ****************************************************************************
659  // Update pressure variable with phi -- note that phi is dt * change in pressure
660  // ****************************************************************************
661  MultiFab::Saxpy(pp_inc[lev], 1.0/l_dt, phi_lev,0,0,1,1);
662 }
void ConvertForProjection(const MultiFab &den_div, const MultiFab &den_mlt, MultiFab &xmom, MultiFab &ymom, MultiFab &zmom, const Box &domain, const Vector< BCRec > &domain_bcs_type_h)
Definition: ERF_ConvertForProjection.cpp:25
void enforceInOutSolvability(int, Array< MultiFab *, AMREX_SPACEDIM > &vels_vec, Array< MultiFab *, AMREX_SPACEDIM > &area_vec, const Geometry &geom)
Definition: ERF_ConvertForProjection.cpp:326
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real OmegaFromW(int &i, int &j, int &k, amrex::Real w, const amrex::Array4< const amrex::Real > &u_arr, const amrex::Array4< const amrex::Real > &v_arr, const amrex::Array4< const amrex::Real > &mf_u, const amrex::Array4< const amrex::Real > &mf_v, const amrex::Array4< const amrex::Real > &z_nd, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &dxInv)
Definition: ERF_TerrainMetrics.H:412
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real WFromOmega(int &i, int &j, int &k, amrex::Real omega, const amrex::Array4< const amrex::Real > &u_arr, const amrex::Array4< const amrex::Real > &v_arr, const amrex::Array4< const amrex::Real > &mf_u, const amrex::Array4< const amrex::Real > &mf_v, const amrex::Array4< const amrex::Real > &z_nd, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &dxInv)
Definition: ERF_TerrainMetrics.H:462
static bool use_fft
Definition: ERF.H:1197
void solve_with_gmres(int lev, const amrex::Box &subdomain, amrex::MultiFab &rhs, amrex::MultiFab &p, amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &fluxes, amrex::MultiFab &ax_sub, amrex::MultiFab &ay_sub, amrex::MultiFab &az_sub, amrex::MultiFab &, amrex::MultiFab &znd_sub)
Definition: ERF_SolveWithGMRES.cpp:12
void compute_divergence(int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::Geometry const &geom_at_lev)
Definition: ERF_ComputeDivergence.cpp:10
void solve_with_mlmg(int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
Definition: ERF_SolveWithMLMG.cpp:40
void solve_with_EB_mlmg(int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
Definition: ERF_SolveWithEBMLMG.cpp:19
amrex::Real volWgtSumMF(int lev, const amrex::MultiFab &mf, int comp, bool finemask)
Definition: ERF_VolWgtSum.cpp:17
@ omega
Definition: ERF_Morrison.H:53
integer, private isub
Definition: ERF_module_mp_morr_two_moment.F90:164
Here is the call graph for this function:

◆ project_velocity()

void ERF::project_velocity ( int  lev,
amrex::Real  dt 
)

Project the single-level velocity field to enforce the anelastic constraint Note that the level may or may not be level 0.

11 {
12  // Impose FillBoundary on density since we use it in the conversion of velocity to momentum
13  vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
14 
15  BL_PROFILE("ERF::project_velocity()");
16  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
17  vars_new[lev][Vars::yvel], IntVect{0},
18  vars_new[lev][Vars::zvel], IntVect{0},
19  vars_new[lev][Vars::cons],
20  rU_new[lev], rV_new[lev], rW_new[lev],
21  Geom(lev).Domain(), domain_bcs_type);
22 
23  Vector<MultiFab> tmp_mom;
24 
25  tmp_mom.push_back(MultiFab(vars_new[lev][Vars::cons],make_alias,0,1));
26  tmp_mom.push_back(MultiFab(rU_new[lev],make_alias,0,1));
27  tmp_mom.push_back(MultiFab(rV_new[lev],make_alias,0,1));
28  tmp_mom.push_back(MultiFab(rW_new[lev],make_alias,0,1));
29 
30  project_momenta(lev, l_dt, tmp_mom);
31 
33  vars_new[lev][Vars::yvel],
34  vars_new[lev][Vars::zvel],
35  vars_new[lev][Vars::cons],
36  rU_new[lev], rV_new[lev], rW_new[lev],
37  Geom(lev).Domain(), domain_bcs_type);
38  }
void project_momenta(int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars)
Definition: ERF_PoissonSolve.cpp:44
Here is the call graph for this function:

◆ project_velocity_tb()

void ERF::project_velocity_tb ( int  lev,
amrex::Real  dt,
amrex::Vector< amrex::MultiFab > &  vars 
)

Project the single-level velocity field to enforce incompressibility with a thin body

21 {
22  BL_PROFILE("ERF::project_velocity_tb()");
23  AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::ConstantDz);
24 
25  // Make sure the solver only sees the levels over which we are solving
26  Vector<BoxArray> ba_tmp; ba_tmp.push_back(vmf[Vars::cons].boxArray());
27  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(vmf[Vars::cons].DistributionMap());
28  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
29 
30  // Use the default settings
31  LPInfo info;
32  std::unique_ptr<MLPoisson> p_mlpoisson;
33 #if 0
34  if (overset_imask[0]) {
35  // Add overset mask around thin body
36  p_mlpoisson = std::make_unique<MLPoisson>(geom, grids, dmap, GetVecOfConstPtrs(overset_imask), info);
37  }
38  else
39 #endif
40  {
41  // Use the default settings
42  p_mlpoisson = std::make_unique<MLPoisson>(geom_tmp, ba_tmp, dm_tmp, info);
43  }
44 
45  auto bclo = get_projection_bc(Orientation::low);
46  auto bchi = get_projection_bc(Orientation::high);
47  bool need_adjust_rhs = (projection_has_dirichlet(bclo) || projection_has_dirichlet(bchi)) ? false : true;
48  p_mlpoisson->setDomainBC(bclo, bchi);
49 
50  if (lev > 0) {
51  p_mlpoisson->setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
52  }
53 
54  p_mlpoisson->setLevelBC(0, nullptr);
55 
56  Vector<MultiFab> rhs;
57  Vector<MultiFab> phi;
58  Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
59  Vector<Array<MultiFab,AMREX_SPACEDIM> > deltaf; // f^* - f^{n-1}
60  Vector<Array<MultiFab,AMREX_SPACEDIM> > u_plus_dtdf; // u + dt*deltaf
61 
62  // Used to pass array of const MFs to ComputeDivergence
63  Array<MultiFab const*, AMREX_SPACEDIM> u;
64 
65  rhs.resize(1);
66  phi.resize(1);
67  fluxes.resize(1);
68  deltaf.resize(1);
69  u_plus_dtdf.resize(1);
70 
71  rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
72  phi[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
73  rhs[0].setVal(0.0);
74  phi[0].setVal(0.0);
75 
76  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
77  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
78  u_plus_dtdf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
79 
80  deltaf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
81  deltaf[0][idim].setVal(0.0); // start with f^* == f^{n-1}
82  }
83 
84 #if 0
85  // DEBUG
86  u[0] = &(vmf[Vars::xvel]);
87  u[1] = &(vmf[Vars::yvel]);
88  u[2] = &(vmf[Vars::zvel]);
89  computeDivergence(rhs[0], u, geom[0]);
90  Print() << "Max norm of divergence before solve at level 0 : " << rhs[0].norm0() << std::endl;
91 #endif
92 
93  for (int itp = 0; itp < solverChoice.ncorr; ++itp)
94  {
95  // Calculate u + dt*deltaf
96  for (int idim = 0; idim < 3; ++idim) {
97  MultiFab::Copy(u_plus_dtdf[0][idim], deltaf[0][idim], 0, 0, 1, 0);
98  u_plus_dtdf[0][0].mult(-l_dt,0,1,0);
99  }
100  MultiFab::Add(u_plus_dtdf[0][0], vmf[Vars::xvel], 0, 0, 1, 0);
101  MultiFab::Add(u_plus_dtdf[0][1], vmf[Vars::yvel], 0, 0, 1, 0);
102  MultiFab::Add(u_plus_dtdf[0][2], vmf[Vars::zvel], 0, 0, 1, 0);
103 
104  u[0] = &(u_plus_dtdf[0][0]);
105  u[1] = &(u_plus_dtdf[0][1]);
106  u[2] = &(u_plus_dtdf[0][2]);
107  computeDivergence(rhs[0], u, geom_tmp[0]);
108 
109 #if 0
110  // DEBUG
111  if (itp==0) {
112  for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
113  {
114  const Box& bx = mfi.tilebox();
115  const Array4<Real const>& divU = rhs[0].const_array(mfi);
116  const Array4<Real const>& uarr = vmf[Vars::xvel].const_array(mfi);
117  const Array4<Real const>& varr = vmf[Vars::yvel].const_array(mfi);
118  const Array4<Real const>& warr = vmf[Vars::zvel].const_array(mfi);
119  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
120  {
121  if ((i>=120) && (i<=139) && (j==0) && ((k>=127)&&(k<=128))) {
122  amrex::AllPrint() << "before project div"<<IntVect(i,j,k)<<" = "<< divU(i,j,k)
123  << " u: " << uarr(i,j,k) << " " << uarr(i+1,j,k)
124  << " v: " << varr(i,j,k) << " " << varr(i,j+1,k)
125  << " w: " << warr(i,j,k) << " " << warr(i,j,k+1)
126  << std::endl;
127  }
128  });
129  }
130  }
131 #endif
132 
133  // If all Neumann BCs, adjust RHS to make sure we can converge
134  if (need_adjust_rhs) {
135  Real offset = volWgtSumMF(lev, rhs[0], 0, false);
136  // amrex::Print() << "Poisson solvability offset = " << offset << std::endl;
137  rhs[0].plus(-offset, 0, 1);
138  }
139 
140  // Initialize phi to 0
141  phi[0].setVal(0.0);
142 
143  MLMG mlmg(*p_mlpoisson);
144  int max_iter = 100;
145  mlmg.setMaxIter(max_iter);
146 
147  mlmg.setVerbose(mg_verbose);
148  //mlmg.setBottomVerbose(mg_verbose);
149 
150  // solve for dt*p
151  mlmg.solve(GetVecOfPtrs(phi),
152  GetVecOfConstPtrs(rhs),
155 
156  mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
157 
158  // Calculate new intermediate body force with updated gradp
159  if (thin_xforce[lev]) {
160  MultiFab::Copy( deltaf[0][0], fluxes[0][0], 0, 0, 1, 0);
161  ApplyInvertedMask(deltaf[0][0], *xflux_imask[0]);
162  }
163  if (thin_yforce[lev]) {
164  MultiFab::Copy( deltaf[0][1], fluxes[0][1], 0, 0, 1, 0);
165  ApplyInvertedMask(deltaf[0][1], *yflux_imask[0]);
166  }
167  if (thin_zforce[lev]) {
168  MultiFab::Copy( deltaf[0][2], fluxes[0][2], 0, 0, 1, 0);
169  ApplyInvertedMask(deltaf[0][2], *zflux_imask[0]);
170  }
171 
172  // DEBUG
173  // for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
174  // {
175  // const Box& bx = mfi.tilebox();
176  // const Array4<Real const>& dfz_arr = deltaf[0][2].const_array(mfi);
177  // ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
178  // {
179  // if ((i>=120) && (i<=139) && (j==0) && (k==128)) {
180  // amrex::AllPrint()
181  // << " piter" << itp
182  // << " dfz"<<IntVect(i,j,k)<<" = "<< dfz_arr(i,j,k)
183  // << std::endl;
184  // }
185  // });
186  // }
187 
188  // Update pressure variable with phi -- note that phi is change in pressure, not the full pressure
189  MultiFab::Saxpy(pp_inc[lev], 1.0, phi[0],0,0,1,0);
190 
191  // Subtract grad(phi) from the velocity components
192  Real beta = 1.0;
193  MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
194  MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
195  MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
196  if (thin_xforce[lev]) {
197  ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
198  }
199  if (thin_yforce[lev]) {
200  ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
201  }
202  if (thin_zforce[lev]) {
203  ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
204  }
205  } // itp: pressure-force iterations
206 
207  // ****************************************************************************
208  // Define gradp from fluxes -- note that fluxes is dt * change in Gp
209  // ****************************************************************************
210  MultiFab::Saxpy(gradp[lev][GpVars::gpx],-1.0/l_dt,fluxes[0][0],0,0,1,0);
211  MultiFab::Saxpy(gradp[lev][GpVars::gpy],-1.0/l_dt,fluxes[0][1],0,0,1,0);
212  MultiFab::Saxpy(gradp[lev][GpVars::gpz],-1.0/l_dt,fluxes[0][2],0,0,1,0);
213 
214  gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
215  gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
216  gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
217 
218  // Subtract grad(phi) from the velocity components
219 // Real beta = 1.0;
220 // for (int ilev = lev_min; ilev <= lev_max; ++ilev) {
221 // MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
222 // MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
223 // MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
224 // if (thin_xforce[lev]) {
225 // ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
226 // }
227 // if (thin_yforce[lev]) {
228 // ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
229 // }
230 // if (thin_zforce[lev]) {
231 // ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
232 // }
233 // }
234 
235 #if 0
236  // Confirm that the velocity is now divergence free
237  u[0] = &(vmf[Vars::xvel]);
238  u[1] = &(vmf[Vars::yvel]);
239  u[2] = &(vmf[Vars::zvel]);
240  computeDivergence(rhs[0], u, geom_tmp[0]);
241  Print() << "Max norm of divergence after solve at level " << lev << " : " << rhs[0].norm0() << std::endl;
242 
243 #endif
244 }
AMREX_FORCE_INLINE IntVect offset(const int face_dir, const int normal)
Definition: ERF_ReadBndryPlanes.cpp:28
AMREX_GPU_HOST AMREX_FORCE_INLINE void ApplyInvertedMask(amrex::MultiFab &dst, const amrex::iMultiFab &imask, const int nghost=0)
Definition: ERF_Utils.H:430
amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > get_projection_bc(amrex::Orientation::Side side) const noexcept
Definition: ERF_SolveWithMLMG.cpp:17
bool projection_has_dirichlet(amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > bcs) const
Definition: ERF_PoissonSolve_tb.cpp:8
int ncorr
Definition: ERF_DataStruct.H:944
Here is the call graph for this function:

◆ projection_has_dirichlet()

bool ERF::projection_has_dirichlet ( amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM >  bcs) const
9 {
10  for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
11  if (bcs[dir] == LinOpBCType::Dirichlet) return true;
12  }
13  return false;
14 }

◆ ReadCheckpointFile()

void ERF::ReadCheckpointFile ( )

ERF function for reading data from a checkpoint file during restart.

448 {
449  Print() << "Restart from native checkpoint " << restart_chkfile << "\n";
450 
451  // Header
452  std::string File(restart_chkfile + "/Header");
453 
454  VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
455 
456  Vector<char> fileCharPtr;
457  ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
458  std::string fileCharPtrString(fileCharPtr.dataPtr());
459  std::istringstream is(fileCharPtrString, std::istringstream::in);
460 
461  std::string line, word;
462 
463  int chk_ncomp_cons, chk_ncomp;
464 
465  // read in title line
466  std::getline(is, line);
467 
468  // read in finest_level
469  is >> finest_level;
470  GotoNextLine(is);
471 
472  // read the number of components
473  // for each variable we store
474 
475  // conservative, cell-centered vars
476  is >> chk_ncomp_cons;
477  GotoNextLine(is);
478 
479  // x-velocity on faces
480  is >> chk_ncomp;
481  GotoNextLine(is);
482  AMREX_ASSERT(chk_ncomp == 1);
483 
484  // y-velocity on faces
485  is >> chk_ncomp;
486  GotoNextLine(is);
487  AMREX_ASSERT(chk_ncomp == 1);
488 
489  // z-velocity on faces
490  is >> chk_ncomp;
491  GotoNextLine(is);
492  AMREX_ASSERT(chk_ncomp == 1);
493 
494  // read in array of istep
495  std::getline(is, line);
496  {
497  std::istringstream lis(line);
498  int i = 0;
499  while (lis >> word) {
500  istep[i++] = std::stoi(word);
501  }
502  }
503 
504  // read in array of dt
505  std::getline(is, line);
506  {
507  std::istringstream lis(line);
508  int i = 0;
509  while (lis >> word) {
510  dt[i++] = std::stod(word);
511  }
512  }
513 
514  // read in array of t_new
515  std::getline(is, line);
516  {
517  std::istringstream lis(line);
518  int i = 0;
519  while (lis >> word) {
520  t_new[i++] = std::stod(word);
521  }
522  }
523 
524  for (int lev = 0; lev <= finest_level; ++lev) {
525  // read in level 'lev' BoxArray from Header
526  BoxArray ba;
527  ba.readFrom(is);
528  GotoNextLine(is);
529 
530  // create a distribution mapping
531  DistributionMapping dm { ba, ParallelDescriptor::NProcs() };
532 
533  MakeNewLevelFromScratch (lev, t_new[lev], ba, dm);
534  }
535 
536  // ncomp is only valid after we MakeNewLevelFromScratch (asks micro how many vars)
537  // NOTE: Data is written over ncomp, so check that we match the header file
538  int ncomp_cons = vars_new[0][Vars::cons].nComp();
539 
540  // NOTE: QKE was removed so this is for backward compatibility
541  AMREX_ASSERT((chk_ncomp_cons==ncomp_cons) || ((chk_ncomp_cons-1)==ncomp_cons));
542  //
543  // See if we have a written separate file that tells how many components and how many ghost cells
544  // we have of the base state
545  //
546  // If we can't find the file, then set the number of components to the original number = 3
547  //
548  int ncomp_base_to_read = 3;
549  IntVect ng_base = IntVect{1};
550  {
551  std::string BaseStateFile(restart_chkfile + "/num_base_state_comps");
552 
553  if (amrex::FileExists(BaseStateFile))
554  {
555  Vector<char> BaseStatefileCharPtr;
556  ParallelDescriptor::ReadAndBcastFile(BaseStateFile, BaseStatefileCharPtr);
557  std::string BaseStatefileCharPtrString(BaseStatefileCharPtr.dataPtr());
558 
559  // We set this to the default value of 3 but allow it be larger if th0 and qv0 were written
560  std::istringstream isb(BaseStatefileCharPtrString, std::istringstream::in);
561  isb >> ncomp_base_to_read;
562  isb >> ng_base;
563  }
564  }
565 
566  // read in the MultiFab data
567  for (int lev = 0; lev <= finest_level; ++lev)
568  {
569  // NOTE: For backward compatibility (chk file has QKE)
570  if ((chk_ncomp_cons-1)==ncomp_cons) {
571  MultiFab cons(grids[lev],dmap[lev],chk_ncomp_cons,0);
572  VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
573 
574  // Copy up to RhoKE_comp
575  MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,(RhoKE_comp+1),0);
576 
577  // Only if we have a PBL model do we need to copy QKE is src to KE in dst
578  if ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
579  (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF) ) {
580  MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+1),RhoKE_comp,1,0);
581  vars_new[lev][Vars::cons].mult(0.5,RhoKE_comp,1,0);
582  }
583 
584  // Copy other components
585  int ncomp_remainder = ncomp_cons - (RhoKE_comp + 1);
586  MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+2),(RhoKE_comp+1),ncomp_remainder,0);
587 
588  vars_new[lev][Vars::cons].setBndry(1.0e34);
589  } else {
590  MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
591  VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
592  MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,ncomp_cons,0);
593  vars_new[lev][Vars::cons].setBndry(1.0e34);
594  }
595 
596  MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
597  VisMF::Read(xvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace"));
598  MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
599  vars_new[lev][Vars::xvel].setBndry(1.0e34);
600 
601  MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
602  VisMF::Read(yvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace"));
603  MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
604  vars_new[lev][Vars::yvel].setBndry(1.0e34);
605 
606  MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
607  VisMF::Read(zvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace"));
608  MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
609  vars_new[lev][Vars::zvel].setBndry(1.0e34);
610 
611  if (solverChoice.anelastic[lev] == 1) {
612  MultiFab ppinc(grids[lev],dmap[lev],1,0);
613  VisMF::Read(ppinc, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PP_Inc"));
614  MultiFab::Copy(pp_inc[lev],ppinc,0,0,1,0);
615  pp_inc[lev].FillBoundary(geom[lev].periodicity());
616 
617  MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
618  VisMF::Read(gpx, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpx"));
619  MultiFab::Copy(gradp[lev][GpVars::gpx],gpx,0,0,1,0);
620  gradp[lev][GpVars::gpx].FillBoundary(geom[lev].periodicity());
621 
622  MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
623  VisMF::Read(gpy, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpy"));
624  MultiFab::Copy(gradp[lev][GpVars::gpy],gpy,0,0,1,0);
625  gradp[lev][GpVars::gpy].FillBoundary(geom[lev].periodicity());
626 
627  MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
628  VisMF::Read(gpz, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpz"));
629  MultiFab::Copy(gradp[lev][GpVars::gpz],gpz,0,0,1,0);
630  gradp[lev][GpVars::gpz].FillBoundary(geom[lev].periodicity());
631  }
632 
633  // Note that we read the ghost cells of the base state (unlike above)
634 
635  // The original base state only had 3 components and 1 ghost cell -- we read this
636  // here to be consistent with the old style
637  MultiFab base(grids[lev],dmap[lev],ncomp_base_to_read,ng_base);
638  VisMF::Read(base, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "BaseState"));
639 
640  MultiFab::Copy(base_state[lev],base,0,0,ncomp_base_to_read,ng_base);
641 
642  // Create theta0 from p0, rh0
643  if (ncomp_base_to_read < 4) {
644  for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
645  {
646  // We only compute theta_0 on valid cells since we will impose domain BC's after restart
647  const Box& bx = mfi.tilebox();
648  Array4<Real> const& fab = base_state[lev].array(mfi);
649  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
650  {
652  / fab(i,j,k,BaseState::r0_comp);
653  });
654  }
655  }
656  // Default theta0 to 0
657  if (ncomp_base_to_read < 5) {
658  for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
659  {
660  // We only compute theta_0 on valid cells since we will impose domain BC's after restart
661  const Box& bx = mfi.tilebox();
662  Array4<Real> const& fab = base_state[lev].array(mfi);
663  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
664  {
665  fab(i,j,k,BaseState::qv0_comp) = 0.0;
666  });
667  }
668  }
669  base_state[lev].FillBoundary(geom[lev].periodicity());
670 
671  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
672  // Note that we also read the ghost cells of z_phys_nd
673  IntVect ng = z_phys_nd[lev]->nGrowVect();
674  MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
675  VisMF::Read(z_height, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z_Phys_nd"));
676  MultiFab::Copy(*z_phys_nd[lev],z_height,0,0,1,ng);
678 
679  // Compute the min dz and pass to the micro model
680  Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
681  micro->Set_dzmin(lev, dzmin);
682 
683  if (SolverChoice::mesh_type == MeshType::VariableDz) {
684  MultiFab z_slab(convert(ba2d[lev],IntVect(1,1,1)),dmap[lev],1,0);
685  int klo = geom[lev].Domain().smallEnd(2);
686  for (MFIter mfi(z_slab); mfi.isValid(); ++mfi) {
687  Box nbx = mfi.tilebox();
688  Array4<Real const> const& z_arr = z_phys_nd[lev]->const_array(mfi);
689  Array4<Real > const& z_slab_arr = z_slab.array(mfi);
690  ParallelFor(nbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
691  {
692  z_slab_arr(i,j,k) = z_arr(i,j,klo);
693  });
694  }
695  Real z_min = z_slab.min(0);
696  Real z_max = z_slab.max(0);
697 
698  auto dz = geom[lev].CellSize()[2];
699  if (z_max - z_min < 1.e-8 * dz) {
700  SolverChoice::set_mesh_type(MeshType::StretchedDz);
701  if (verbose > 0) {
702  amrex::Print() << "Resetting mesh type to StretchedDz since terrain is flat" << std::endl;
703  }
704  }
705  }
706  }
707 
708  // Read in the moisture model restart variables
709  std::vector<int> qmoist_indices;
710  std::vector<std::string> qmoist_names;
711  micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
712  int qmoist_nvar = qmoist_indices.size();
713  for (int var = 0; var < qmoist_nvar; var++) {
714  const int ncomp = 1;
715  IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
716  MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
717  VisMF::Read(moist_vars, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", qmoist_names[var]));
718  MultiFab::Copy(*(qmoist[lev][qmoist_indices[var]]),moist_vars,0,0,ncomp,ng_moist);
719  }
720 
721 #if defined(ERF_USE_WINDFARM)
722  if(solverChoice.windfarm_type == WindFarmType::Fitch or
723  solverChoice.windfarm_type == WindFarmType::EWP or
724  solverChoice.windfarm_type == WindFarmType::SimpleAD){
725  IntVect ng = Nturb[lev].nGrowVect();
726  MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng);
727  VisMF::Read(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "NumTurb"));
728  MultiFab::Copy(Nturb[lev],mf_Nturb,0,0,1,ng);
729  }
730 #endif
731 
732  if (solverChoice.lsm_type != LandSurfaceType::None) {
733  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
734  BoxArray ba = lsm_data[lev][mvar]->boxArray();
735  DistributionMapping dm = lsm_data[lev][mvar]->DistributionMap();
736  IntVect ng = lsm_data[lev][mvar]->nGrowVect();
737  int nvar = lsm_data[lev][mvar]->nComp();
738  MultiFab lsm_vars(ba,dm,nvar,ng);
739  VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmVars"));
740  MultiFab::Copy(*(lsm_data[lev][mvar]),lsm_vars,0,0,nvar,ng);
741  }
742  }
743 
744 
745  IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
746  MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
747 
748  std::string MapFacMFileName(restart_chkfile + "/Level_0/MapFactor_mx_H");
749  if (amrex::FileExists(MapFacMFileName)) {
750  VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_mx"));
751  } else {
752  VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_m"));
753  }
754  MultiFab::Copy(*mapfac[lev][MapFacType::m_x],mf_m,0,0,1,ng);
755 
756 #if 0
758  VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_my"));
759  MultiFab::Copy(*mapfac[lev][MapFacType::m_y],mf_m,0,0,1,ng);
760  }
761 #endif
762 
763  ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
764  MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
765 
766  std::string MapFacUFileName(restart_chkfile + "/Level_0/MapFactor_ux_H");
767  if (amrex::FileExists(MapFacUFileName)) {
768  VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_ux"));
769  } else {
770  VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_u"));
771  }
772  MultiFab::Copy(*mapfac[lev][MapFacType::u_x],mf_u,0,0,1,ng);
773 
774 #if 0
776  VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_uy"));
777  MultiFab::Copy(*mapfac[lev][MapFacType::u_y],mf_u,0,0,1,ng);
778  }
779 #endif
780 
781  ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
782  MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
783 
784  std::string MapFacVFileName(restart_chkfile + "/Level_0/MapFactor_vx_H");
785  if (amrex::FileExists(MapFacVFileName)) {
786  VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vx"));
787  } else {
788  VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_v"));
789  }
790  MultiFab::Copy(*mapfac[lev][MapFacType::v_x],mf_v,0,0,1,ng);
791 
792 #if 0
794  VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vy"));
795  MultiFab::Copy(*mapfac[lev][MapFacType::v_y],mf_v,0,0,1,ng);
796  }
797 #endif
798 
799 
800  // NOTE: We read MOST data in ReadCheckpointFileMOST (see below)!
801 
802  // See if we wrote out SST data
803  std::string FirstSSTFileName(restart_chkfile + "/Level_0/SST_0_H");
804  if (amrex::FileExists(FirstSSTFileName))
805  {
806  amrex::Print() << "Reading SST data" << std::endl;
807  int ntimes = 1;
808  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
809  MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
810  sst_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
811  for (int nt(0); nt<ntimes; ++nt) {
812  VisMF::Read(sst_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
813  "SST_" + std::to_string(nt)));
814  MultiFab::Copy(*sst_lev[lev][nt],sst_at_t,0,0,1,ng);
815  }
816  }
817 
818  // See if we wrote out TSK data
819  std::string FirstTSKFileName(restart_chkfile + "/Level_0/TSK_0_H");
820  if (amrex::FileExists(FirstTSKFileName))
821  {
822  amrex::Print() << "Reading TSK data" << std::endl;
823  int ntimes = 1;
824  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
825  MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
826  tsk_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
827  for (int nt(0); nt<ntimes; ++nt) {
828  VisMF::Read(tsk_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
829  "TSK_" + std::to_string(nt)));
830  MultiFab::Copy(*tsk_lev[lev][nt],tsk_at_t,0,0,1,ng);
831  }
832  }
833 
834  std::string LMaskFileName(restart_chkfile + "/Level_0/LMASK_0_H");
835  if (amrex::FileExists(LMaskFileName))
836  {
837  amrex::Print() << "Reading LMASK data" << std::endl;
838  int ntimes = 1;
839  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
840  MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
841  for (int nt(0); nt<ntimes; ++nt) {
842  VisMF::Read(lmask_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
843  "LMASK_" + std::to_string(nt)));
844  for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
845  const Box& bx = mfi.growntilebox();
846  Array4<int> const& dst_arr = lmask_lev[lev][nt]->array(mfi);
847  Array4<Real> const& src_arr = lmask_at_t.array(mfi);
848  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
849  {
850  dst_arr(i,j,k) = int(src_arr(i,j,k));
851  });
852  }
853  }
854  } else {
855  // Allow idealized cases over water, used to set lmask
856  ParmParse pp("erf");
857  int is_land;
858  if (pp.query("is_land", is_land, lev)) {
859  if (is_land == 1) {
860  amrex::Print() << "Level " << lev << " is land" << std::endl;
861  } else if (is_land == 0) {
862  amrex::Print() << "Level " << lev << " is water" << std::endl;
863  } else {
864  Error("is_land should be 0 or 1");
865  }
866  lmask_lev[lev][0]->setVal(is_land);
867  } else {
868  // Default to land everywhere if not specified
869  lmask_lev[lev][0]->setVal(1);
870  }
871  lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
872  }
873 
874  IntVect ngv = ng; ngv[2] = 0;
875 
876  // Read lat/lon if it exists
878  amrex::Print() << "Reading Lat/Lon variables" << std::endl;
879  MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
880  MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
881  VisMF::Read(lat, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LAT"));
882  VisMF::Read(lon, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LON"));
883  lat_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
884  lon_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
885  MultiFab::Copy(*lat_m[lev],lat,0,0,1,ngv);
886  MultiFab::Copy(*lon_m[lev],lon,0,0,1,ngv);
887  }
888 
889 #ifdef ERF_USE_NETCDF
890  // Read sinPhi and cosPhi if it exists
892  amrex::Print() << "Reading Coriolis factors" << std::endl;
893  MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
894  MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
895  VisMF::Read(sphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "SinPhi"));
896  VisMF::Read(cphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "CosPhi"));
897  sinPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
898  cosPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
899  MultiFab::Copy(*sinPhi_m[lev],sphi,0,0,1,ngv);
900  MultiFab::Copy(*cosPhi_m[lev],cphi,0,0,1,ngv);
901  }
902 
903  if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
904 
905  if (lev == 0) {
906  MultiFab tmp1d(ba1d[0],dmap[0],1,0);
907 
908  VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C1H"));
909  MultiFab::Copy(*mf_C1H,tmp1d,0,0,1,0);
910 
911  VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C2H"));
912  MultiFab::Copy(*mf_C2H,tmp1d,0,0,1,0);
913 
914  MultiFab tmp2d(ba2d[0],dmap[0],1,mf_MUB->nGrowVect());
915 
916  VisMF::Read(tmp2d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MUB"));
917  MultiFab::Copy(*mf_MUB,tmp2d,0,0,1,mf_MUB->nGrowVect());
918  }
919  }
920 #endif
921 
922  } // for lev
923 
924 #ifdef ERF_USE_PARTICLES
925  restartTracers((ParGDBBase*)GetParGDB(),restart_chkfile);
926  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
927  dynamic_cast<LagrangianMicrophysics&>(*micro).restartParticles((ParGDBBase*)GetParGDB(),restart_chkfile);
928  }
929 #endif
930 
931 #if 0
932 #ifdef ERF_USE_NETCDF
933  // Read bdy_data files
934  if ( ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
936  {
937  int ioproc = ParallelDescriptor::IOProcessorNumber(); // I/O rank
938  int num_time;
939  int num_var;
940  Vector<Box> bx_v;
941  if (ParallelDescriptor::IOProcessor()) {
942  // Open header file and read from it
943  std::ifstream bdy_h_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_H"));
944  bdy_h_file >> num_time;
945  bdy_h_file >> num_var;
946  bdy_h_file >> start_bdy_time;
947  bdy_h_file >> bdy_time_interval;
948  bdy_h_file >> real_width;
949  bx_v.resize(4*num_var);
950  for (int ivar(0); ivar<num_var; ++ivar) {
951  bdy_h_file >> bx_v[4*ivar ];
952  bdy_h_file >> bx_v[4*ivar+1];
953  bdy_h_file >> bx_v[4*ivar+2];
954  bdy_h_file >> bx_v[4*ivar+3];
955  }
956 
957  // IO size the FABs
958  bdy_data_xlo.resize(num_time);
959  bdy_data_xhi.resize(num_time);
960  bdy_data_ylo.resize(num_time);
961  bdy_data_yhi.resize(num_time);
962  for (int itime(0); itime<num_time; ++itime) {
963  bdy_data_xlo[itime].resize(num_var);
964  bdy_data_xhi[itime].resize(num_var);
965  bdy_data_ylo[itime].resize(num_var);
966  bdy_data_yhi[itime].resize(num_var);
967  for (int ivar(0); ivar<num_var; ++ivar) {
968  bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar ]);
969  bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
970  bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
971  bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
972  }
973  }
974 
975  // Open data file and read from it
976  std::ifstream bdy_d_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_D"));
977  for (int itime(0); itime<num_time; ++itime) {
978  for (int ivar(0); ivar<num_var; ++ivar) {
979  bdy_data_xlo[itime][ivar].readFrom(bdy_d_file);
980  bdy_data_xhi[itime][ivar].readFrom(bdy_d_file);
981  bdy_data_ylo[itime][ivar].readFrom(bdy_d_file);
982  bdy_data_yhi[itime][ivar].readFrom(bdy_d_file);
983  }
984  }
985  } // IO
986 
987  // Broadcast the data
988  ParallelDescriptor::Barrier();
989  ParallelDescriptor::Bcast(&start_bdy_time,1,ioproc);
990  ParallelDescriptor::Bcast(&bdy_time_interval,1,ioproc);
991  ParallelDescriptor::Bcast(&real_width,1,ioproc);
992  ParallelDescriptor::Bcast(&num_time,1,ioproc);
993  ParallelDescriptor::Bcast(&num_var,1,ioproc);
994 
995  // Everyone size their boxes
996  bx_v.resize(4*num_var);
997 
998  ParallelDescriptor::Bcast(bx_v.dataPtr(),bx_v.size(),ioproc);
999 
1000  // Everyone but IO size their FABs
1001  if (!ParallelDescriptor::IOProcessor()) {
1002  bdy_data_xlo.resize(num_time);
1003  bdy_data_xhi.resize(num_time);
1004  bdy_data_ylo.resize(num_time);
1005  bdy_data_yhi.resize(num_time);
1006  for (int itime(0); itime<num_time; ++itime) {
1007  bdy_data_xlo[itime].resize(num_var);
1008  bdy_data_xhi[itime].resize(num_var);
1009  bdy_data_ylo[itime].resize(num_var);
1010  bdy_data_yhi[itime].resize(num_var);
1011  for (int ivar(0); ivar<num_var; ++ivar) {
1012  bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar ]);
1013  bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
1014  bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
1015  bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
1016  }
1017  }
1018  }
1019 
1020  for (int itime(0); itime<num_time; ++itime) {
1021  for (int ivar(0); ivar<num_var; ++ivar) {
1022  ParallelDescriptor::Bcast(bdy_data_xlo[itime][ivar].dataPtr(),bdy_data_xlo[itime][ivar].box().numPts(),ioproc);
1023  ParallelDescriptor::Bcast(bdy_data_xhi[itime][ivar].dataPtr(),bdy_data_xhi[itime][ivar].box().numPts(),ioproc);
1024  ParallelDescriptor::Bcast(bdy_data_ylo[itime][ivar].dataPtr(),bdy_data_ylo[itime][ivar].box().numPts(),ioproc);
1025  ParallelDescriptor::Bcast(bdy_data_yhi[itime][ivar].dataPtr(),bdy_data_yhi[itime][ivar].box().numPts(),ioproc);
1026  }
1027  }
1028  } // init_type == WRFInput or Metgrid
1029 #endif
1030 #endif
1031 }
struct @19 in
static void GotoNextLine(std::istream &is)
Definition: ERF_Checkpoint.cpp:16
void MakeNewLevelFromScratch(int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
Definition: ERF_MakeNewLevel.cpp:25
bool variable_coriolis
Definition: ERF_DataStruct.H:1040
bool has_lat_lon
Definition: ERF_DataStruct.H:1039
static void set_mesh_type(MeshType new_mesh_type)
Definition: ERF_DataStruct.H:913
Here is the call graph for this function:

◆ ReadCheckpointFileSurfaceLayer()

void ERF::ReadCheckpointFileSurfaceLayer ( )

ERF function for reading additional data for MOST from a checkpoint file during restart.

This is called after the ABLMost object is instantiated.

1040 {
1041  for (int lev = 0; lev <= finest_level; ++lev)
1042  {
1043  amrex::Print() << "Reading MOST variables" << std::endl;
1044 
1045  IntVect ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
1046  MultiFab m_var(ba2d[lev],dmap[lev],1,ng);
1047  MultiFab* dst = nullptr;
1048 
1049  // U*
1050  std::string UstarFileName(restart_chkfile + "/Level_0/Ustar_H");
1051  if (amrex::FileExists(UstarFileName)) {
1052  dst = m_SurfaceLayer->get_u_star(lev);
1053  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Ustar"));
1054  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1055  }
1056 
1057  // W*
1058  std::string WstarFileName(restart_chkfile + "/Level_0/Wstar_H");
1059  if (amrex::FileExists(WstarFileName)) {
1060  dst = m_SurfaceLayer->get_w_star(lev);
1061  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Wstar"));
1062  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1063  }
1064 
1065  // T*
1066  std::string TstarFileName(restart_chkfile + "/Level_0/Tstar_H");
1067  if (amrex::FileExists(TstarFileName)) {
1068  dst = m_SurfaceLayer->get_t_star(lev);
1069  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Tstar"));
1070  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1071  }
1072 
1073  // Q*
1074  std::string QstarFileName(restart_chkfile + "/Level_0/Qstar_H");
1075  if (amrex::FileExists(QstarFileName)) {
1076  dst = m_SurfaceLayer->get_q_star(lev);
1077  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qstar"));
1078  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1079  }
1080 
1081  // Olen
1082  std::string OlenFileName(restart_chkfile + "/Level_0/Olen_H");
1083  if (amrex::FileExists(OlenFileName)) {
1084  dst = m_SurfaceLayer->get_olen(lev);
1085  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Olen"));
1086  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1087  }
1088 
1089  // Qsurf
1090  std::string QsurfFileName(restart_chkfile + "/Level_0/Qsurf_H");
1091  if (amrex::FileExists(QsurfFileName)) {
1092  dst = m_SurfaceLayer->get_q_surf(lev);
1093  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qsurf"));
1094  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1095  }
1096 
1097  // PBLH
1098  std::string PBLHFileName(restart_chkfile + "/Level_0/PBLH_H");
1099  if (amrex::FileExists(PBLHFileName)) {
1100  dst = m_SurfaceLayer->get_pblh(lev);
1101  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PBLH"));
1102  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1103  }
1104 
1105  // Z0
1106  std::string Z0FileName(restart_chkfile + "/Level_0/Z0_H");
1107  if (amrex::FileExists(Z0FileName)) {
1108  dst = m_SurfaceLayer->get_z0(lev);
1109  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z0"));
1110  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1111  }
1112  }
1113 }

◆ ReadParameters()

void ERF::ReadParameters ( )
private
2068 {
2069  {
2070  ParmParse pp; // Traditionally, max_step and stop_time do not have prefix.
2071  pp.query("max_step", max_step);
2072  if (max_step < 0) {
2073  max_step = std::numeric_limits<int>::max();
2074  }
2075 
2076  // TODO: more robust general datetime parsing
2077  std::string start_datetime, stop_datetime;
2078  if (pp.query("start_datetime", start_datetime)) {
2079  if (start_datetime.length() == 16) { // YYYY-MM-DD HH:MM
2080  start_datetime += ":00"; // add seconds
2081  }
2082  if (start_datetime.length() != 19) {
2083  Print() << "Got start_datetime = \"" << start_datetime
2084  << "\", format should be " << datetime_format << std::endl;
2085  exit(0);
2086  }
2087  start_time = getEpochTime(start_datetime, datetime_format);
2088  Print() << "Start datetime : " << start_datetime << std::endl;
2089 
2090  if (pp.query("stop_datetime", stop_datetime)) {
2091  if (stop_datetime.length() == 16) { // YYYY-MM-DD HH:MM
2092  stop_datetime += ":00"; // add seconds
2093  }
2094  if (stop_datetime.length() != 19) {
2095  Print() << "Got stop_datetime = \"" << stop_datetime
2096  << "\", format should be " << datetime_format << std::endl;
2097  exit(0);
2098  }
2099  stop_time = getEpochTime(stop_datetime, datetime_format);
2100  Print() << "Stop datetime : " << start_datetime << std::endl;
2101  } else if (pp.query("stop_time", stop_time)) {
2102  Print() << "Sim length : " << stop_time << " s" << std::endl;
2103  stop_time += start_time;
2104  }
2105 
2106  use_datetime = true;
2107 
2108  } else {
2109  pp.query("stop_time", stop_time);
2110  pp.query("start_time", start_time); // This is optional, it defaults to 0
2111  }
2112  }
2113 
2114  ParmParse pp(pp_prefix);
2115  ParmParse pp_amr("amr");
2116  {
2117  pp.query("regrid_level_0_on_restart", regrid_level_0_on_restart);
2118  pp.query("regrid_int", regrid_int);
2119  pp.query("check_file", check_file);
2120 
2121  // The regression tests use "amr.restart" and "amr.m_check_int" so we allow
2122  // for those or "erf.restart" / "erf.m_check_int" with the former taking
2123  // precedence if both are specified
2124  pp.query("check_int", m_check_int);
2125  pp.query("check_per", m_check_per);
2126  pp_amr.query("check_int", m_check_int);
2127  pp_amr.query("check_per", m_check_per);
2128 
2129  pp.query("restart", restart_chkfile);
2130  pp_amr.query("restart", restart_chkfile);
2131 
2132  // Verbosity
2133  pp.query("v", verbose);
2134  pp.query("mg_v", mg_verbose);
2135  pp.query("use_fft", use_fft);
2136 #ifndef ERF_USE_FFT
2137  if (use_fft) {
2138  amrex::Abort("You must build with USE_FFT in order to set use_fft = true in your inputs file");
2139  }
2140 #endif
2141 
2142  // Check for NaNs?
2143  pp.query("check_for_nans", check_for_nans);
2144 
2145  // Frequency of diagnostic output
2146  pp.query("sum_interval", sum_interval);
2147  pp.query("sum_period" , sum_per);
2148 
2149  pp.query("pert_interval", pert_interval);
2150 
2151  // Time step controls
2152  pp.query("cfl", cfl);
2153  pp.query("substepping_cfl", sub_cfl);
2154  pp.query("init_shrink", init_shrink);
2155  pp.query("change_max", change_max);
2156  pp.query("dt_max_initial", dt_max_initial);
2157  pp.query("dt_max", dt_max);
2158 
2159  fixed_dt.resize(max_level+1,-1.);
2160  fixed_fast_dt.resize(max_level+1,-1.);
2161 
2162  pp.query("fixed_dt", fixed_dt[0]);
2163  pp.query("fixed_fast_dt", fixed_fast_dt[0]);
2164 
2165  int nlevs_max = max_level + 1;
2166  istep.resize(nlevs_max, 0);
2167  nsubsteps.resize(nlevs_max, 1);
2168  // This is the default
2169  for (int lev = 1; lev <= max_level; ++lev) {
2170  nsubsteps[lev] = MaxRefRatio(lev-1);
2171  }
2172 
2173  if (max_level > 0) {
2174  ParmParse pp_erf("erf");
2175  int count = pp_erf.countval("dt_ref_ratio");
2176  if (count > 0) {
2177  Vector<int> nsub;
2178  nsub.resize(nlevs_max, 0);
2179  if (count == 1) {
2180  pp_erf.queryarr("dt_ref_ratio", nsub, 0, 1);
2181  for (int lev = 1; lev <= max_level; ++lev) {
2182  nsubsteps[lev] = nsub[0];
2183  }
2184  } else {
2185  pp_erf.queryarr("dt_ref_ratio", nsub, 0, max_level);
2186  for (int lev = 1; lev <= max_level; ++lev) {
2187  nsubsteps[lev] = nsub[lev-1];
2188  }
2189  }
2190  }
2191  }
2192 
2193  // Make sure we do this after we have defined nsubsteps above
2194  for (int lev = 1; lev <= max_level; lev++)
2195  {
2196  fixed_dt[lev] = fixed_dt[lev-1] / static_cast<Real>(nsubsteps[lev]);
2197  fixed_fast_dt[lev] = fixed_fast_dt[lev-1] / static_cast<Real>(nsubsteps[lev]);
2198  }
2199 
2200  pp.query("fixed_mri_dt_ratio", fixed_mri_dt_ratio);
2201 
2202  // We use this to keep track of how many boxes we read in from WRF initialization
2203  num_files_at_level.resize(max_level+1,0);
2204 
2205  // We use this to keep track of how many boxes are specified thru the refinement indicators
2206  num_boxes_at_level.resize(max_level+1,0);
2207  boxes_at_level.resize(max_level+1);
2208 
2209  // We always have exactly one file at level 0
2210  num_boxes_at_level[0] = 1;
2211  boxes_at_level[0].resize(1);
2212  boxes_at_level[0][0] = geom[0].Domain();
2213 
2214 #ifdef ERF_USE_NETCDF
2215  nc_init_file.resize(max_level+1);
2216 
2217  // NetCDF wrfinput initialization files -- possibly multiple files at each of multiple levels
2218  // but we always have exactly one file at level 0
2219  for (int lev = 0; lev <= max_level; lev++) {
2220  const std::string nc_file_names = Concatenate("nc_init_file_",lev,1);
2221  if (pp.contains(nc_file_names.c_str())) {
2222  int num_files = pp.countval(nc_file_names.c_str());
2223  num_files_at_level[lev] = num_files;
2224  nc_init_file[lev].resize(num_files);
2225  pp.queryarr(nc_file_names.c_str(), nc_init_file[lev],0,num_files);
2226  for (int j = 0; j < num_files; j++) {
2227  Print() << "Reading NC init file names at level " << lev << " and index " << j << " : " << nc_init_file[lev][j] << std::endl;
2228  } // j
2229  } // if pp.contains
2230  } // lev
2231 
2232  // NetCDF wrfbdy lateral boundary file
2233  if (pp.query("nc_bdy_file", nc_bdy_file)) {
2234  Print() << "Reading NC bdy file name " << nc_bdy_file << std::endl;
2235  }
2236 
2237  // NetCDF wrflow lateral boundary file
2238  if (pp.query("nc_low_file", nc_low_file)) {
2239  Print() << "Reading NC low file name " << nc_low_file << std::endl;
2240  }
2241 
2242 #endif
2243 
2244  // Options for vertical interpolation of met_em*.nc data.
2245  pp.query("metgrid_debug_quiescent", metgrid_debug_quiescent);
2246  pp.query("metgrid_debug_isothermal", metgrid_debug_isothermal);
2247  pp.query("metgrid_debug_dry", metgrid_debug_dry);
2248  pp.query("metgrid_debug_psfc", metgrid_debug_psfc);
2249  pp.query("metgrid_debug_msf", metgrid_debug_msf);
2250  pp.query("metgrid_interp_theta", metgrid_interp_theta);
2251  pp.query("metgrid_basic_linear", metgrid_basic_linear);
2252  pp.query("metgrid_use_below_sfc", metgrid_use_below_sfc);
2253  pp.query("metgrid_use_sfc", metgrid_use_sfc);
2254  pp.query("metgrid_retain_sfc", metgrid_retain_sfc);
2255  pp.query("metgrid_proximity", metgrid_proximity);
2256  pp.query("metgrid_order", metgrid_order);
2257  pp.query("metgrid_force_sfc_k", metgrid_force_sfc_k);
2258 
2259  // Set default to FullState for now ... later we will try Perturbation
2260  interpolation_type = StateInterpType::FullState;
2261  pp.query_enum_case_insensitive("interpolation_type" ,interpolation_type);
2262 
2263  PlotFileType plotfile3d_type_temp = PlotFileType::None;
2264  pp.query_enum_case_insensitive("plotfile_type" ,plotfile3d_type_temp);
2265  pp.query_enum_case_insensitive("plotfile_type_1",plotfile3d_type_1);
2266  pp.query_enum_case_insensitive("plotfile_type_2",plotfile3d_type_2);
2267 
2268  PlotFileType plotfile2d_type_temp = PlotFileType::None;
2269  pp.query_enum_case_insensitive("plotfile2d_type" ,plotfile2d_type_temp);
2270  pp.query_enum_case_insensitive("plotfile2d_type_1",plotfile2d_type_1);
2271  pp.query_enum_case_insensitive("plotfile2d_type_2",plotfile2d_type_2);
2272  //
2273  // This option is for backward consistency -- if only plotfile_type is set,
2274  // then it will be used for both 1 and 2 if and only if they are not set
2275  //
2276  // Default is native amrex if no type is specified
2277  //
2278  if (plotfile3d_type_temp == PlotFileType::None) {
2279  if (plotfile3d_type_1 == PlotFileType::None) {
2280  plotfile3d_type_1 = PlotFileType::Amrex;
2281  }
2282  if (plotfile3d_type_2 == PlotFileType::None) {
2283  plotfile3d_type_2 = PlotFileType::Amrex;
2284  }
2285  } else {
2286  if (plotfile3d_type_1 == PlotFileType::None) {
2287  plotfile3d_type_1 = plotfile3d_type_temp;
2288  } else {
2289  amrex::Abort("You must set either plotfile_type or plotfile_type_1, not both");
2290  }
2291  if (plotfile3d_type_2 == PlotFileType::None) {
2292  plotfile3d_type_2 = plotfile3d_type_temp;
2293  } else {
2294  amrex::Abort("You must set either plotfile_type or plotfile_type_2, not both");
2295  }
2296  }
2297  if (plotfile2d_type_temp == PlotFileType::None) {
2298  if (plotfile2d_type_1 == PlotFileType::None) {
2299  plotfile2d_type_1 = PlotFileType::Amrex;
2300  }
2301  if (plotfile2d_type_2 == PlotFileType::None) {
2302  plotfile2d_type_2 = PlotFileType::Amrex;
2303  }
2304  } else {
2305  if (plotfile2d_type_1 == PlotFileType::None) {
2306  plotfile2d_type_1 = plotfile2d_type_temp;
2307  } else {
2308  amrex::Abort("You must set either plotfile2d_type or plotfile2d_type_1, not both");
2309  }
2310  if (plotfile2d_type_2 == PlotFileType::None) {
2311  plotfile2d_type_2 = plotfile2d_type_temp;
2312  } else {
2313  amrex::Abort("You must set either plotfile2d_type or plotfile2d_type_2, not both");
2314  }
2315  }
2316 #ifndef ERF_USE_NETCDF
2317  if (plotfile3d_type_1 == PlotFileType::Netcdf ||
2318  plotfile3d_type_2 == PlotFileType::Netcdf ||
2319  plotfile2d_type_1 == PlotFileType::Netcdf ||
2320  plotfile2d_type_2 == PlotFileType::Netcdf) {
2321  amrex::Abort("Plotfile type = Netcdf is not allowed without USE_NETCDF = TRUE");
2322  }
2323 #endif
2324 
2325  pp.query("plot_file_1" , plot3d_file_1);
2326  pp.query("plot_file_2" , plot3d_file_2);
2327  pp.query("plot2d_file_1", plot2d_file_1);
2328  pp.query("plot2d_file_2", plot2d_file_2);
2329 
2330  pp.query("plot_int_1" , m_plot3d_int_1);
2331  pp.query("plot_int_2" , m_plot3d_int_2);
2332  pp.query("plot_per_1" , m_plot3d_per_1);
2333  pp.query("plot_per_2" , m_plot3d_per_2);
2334 
2335  pp.query("plot2d_int_1" , m_plot2d_int_1);
2336  pp.query("plot2d_int_2" , m_plot2d_int_2);
2337  pp.query("plot2d_per_1", m_plot2d_per_1);
2338  pp.query("plot2d_per_2", m_plot2d_per_2);
2339 
2340  pp.query("subvol_file", subvol_file);
2341  pp.query("subvol_int" , m_subvol_int);
2342  pp.query("subvol_per" , m_subvol_per);
2343  setSubVolVariables("subvol_sampling_vars",subvol3d_var_names);
2344 
2345  pp.query("expand_plotvars_to_unif_rr",m_expand_plotvars_to_unif_rr);
2346 
2347  pp.query("plot_face_vels",m_plot_face_vels);
2348 
2349  if ( (m_plot3d_int_1 > 0 && m_plot3d_per_1 > 0) ||
2350  (m_plot3d_int_2 > 0 && m_plot3d_per_2 > 0.) ) {
2351  Abort("Must choose only one of plot_int or plot_per");
2352  }
2353  if ( (m_plot2d_int_1 > 0 && m_plot2d_per_1 > 0) ||
2354  (m_plot2d_int_2 > 0 && m_plot2d_per_2 > 0.) ) {
2355  Abort("Must choose only one of plot_int or plot_per");
2356  }
2357 
2358  pp.query("profile_int", profile_int);
2359  pp.query("destag_profiles", destag_profiles);
2360 
2361  pp.query("plot_lsm", plot_lsm);
2362 #ifdef ERF_USE_RRTMGP
2363  pp.query("plot_rad", plot_rad);
2364 #endif
2365  pp.query("profile_rad_int", rad_datalog_int);
2366 
2367  pp.query("output_1d_column", output_1d_column);
2368  pp.query("column_per", column_per);
2369  pp.query("column_interval", column_interval);
2370  pp.query("column_loc_x", column_loc_x);
2371  pp.query("column_loc_y", column_loc_y);
2372  pp.query("column_file_name", column_file_name);
2373 
2374  // Sampler output frequency
2375  pp.query("line_sampling_per", line_sampling_per);
2376  pp.query("line_sampling_interval", line_sampling_interval);
2377  pp.query("plane_sampling_per", plane_sampling_per);
2378  pp.query("plane_sampling_interval", plane_sampling_interval);
2379 
2380  // Specify information about outputting planes of data
2381  pp.query("output_bndry_planes", output_bndry_planes);
2382  pp.query("bndry_output_planes_interval", bndry_output_planes_interval);
2383  pp.query("bndry_output_planes_per", bndry_output_planes_per);
2384  pp.query("bndry_output_start_time", bndry_output_planes_start_time);
2385 
2386  // Specify whether ingest boundary planes of data
2387  pp.query("input_bndry_planes", input_bndry_planes);
2388 
2389  // Query the set and total widths for wrfbdy interior ghost cells
2390  pp.query("real_width", real_width);
2391  pp.query("real_set_width", real_set_width);
2392 
2393  // If using real boundaries, do we extrapolate w (or set to 0)
2394  pp.query("real_extrap_w", real_extrap_w);
2395 
2396  // Query the set and total widths for crse-fine interior ghost cells
2397  pp.query("cf_width", cf_width);
2398  pp.query("cf_set_width", cf_set_width);
2399 
2400  // AmrMesh iterate on grids?
2401  bool iterate(true);
2402  pp_amr.query("iterate_grids",iterate);
2403  if (!iterate) SetIterateToFalse();
2404  }
2405 
2406 #ifdef ERF_USE_PARTICLES
2407  readTracersParams();
2408 #endif
2409 
2410  solverChoice.init_params(max_level,pp_prefix);
2411 
2412 #ifndef ERF_USE_NETCDF
2413  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(( (solverChoice.init_type != InitType::WRFInput) &&
2414  (solverChoice.init_type != InitType::Metgrid ) &&
2415  (solverChoice.init_type != InitType::NCFile ) ),
2416  "init_type cannot be 'WRFInput', 'MetGrid' or 'NCFile' if we don't build with netcdf!");
2417 #endif
2418 
2419  // Query the canopy model file name
2420  std::string forestfile;
2421  solverChoice.do_forest_drag = pp.query("forest_file", forestfile);
2423  for (int lev = 0; lev <= max_level; ++lev) {
2424  m_forest_drag[lev] = std::make_unique<ForestDrag>(forestfile);
2425  }
2426  }
2427 
2428  // If init from WRFInput or Metgrid make sure a valid file name is present
2429  if ((solverChoice.init_type == InitType::WRFInput) ||
2430  (solverChoice.init_type == InitType::Metgrid) ||
2431  (solverChoice.init_type == InitType::NCFile) ) {
2432  for (int lev = 0; lev <= max_level; lev++) {
2433  int num_files = nc_init_file[lev].size();
2434  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(num_files>0, "A file name must be present for init type WRFInput, Metgrid or NCFile.");
2435  for (int j = 0; j < num_files; j++) {
2436  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!nc_init_file[lev][j].empty(), "Valid file name must be present for init type WRFInput, Metgrid or NCFile.");
2437  } //j
2438  } // lev
2439  } // InitType
2440 
2441  // What type of land surface model to use
2442  // NOTE: Must be checked after init_params
2443  if (solverChoice.lsm_type == LandSurfaceType::SLM) {
2444  lsm.SetModel<SLM>();
2445  Print() << "SLM land surface model!\n";
2446  } else if (solverChoice.lsm_type == LandSurfaceType::MM5) {
2447  lsm.SetModel<MM5>();
2448  Print() << "MM5 land surface model!\n";
2449 #ifdef ERF_USE_NOAHMP
2450  } else if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
2451  lsm.SetModel<NOAHMP>();
2452  Print() << "Noah-MP land surface model!\n";
2453 #endif
2454  } else if (solverChoice.lsm_type == LandSurfaceType::None) {
2455  lsm.SetModel<NullSurf>();
2456  Print() << "Null land surface model!\n";
2457  } else {
2458  Abort("Dont know this LandSurfaceType!") ;
2459  }
2460 
2461  if (verbose > 0) {
2462  solverChoice.display(max_level,pp_prefix);
2463  }
2464 
2466 }
AMREX_GPU_HOST AMREX_FORCE_INLINE std::time_t getEpochTime(const std::string &dateTime, const std::string &dateTimeFormat)
Definition: ERF_EpochTime.H:15
bool metgrid_basic_linear
Definition: ERF.H:1236
bool metgrid_debug_msf
Definition: ERF.H:1234
std::string plot2d_file_2
Definition: ERF.H:1067
std::string plot3d_file_1
Definition: ERF.H:1064
bool plot_rad
Definition: ERF.H:893
bool m_plot_face_vels
Definition: ERF.H:1080
std::string plot3d_file_2
Definition: ERF.H:1065
int regrid_int
Definition: ERF.H:1057
bool metgrid_retain_sfc
Definition: ERF.H:1239
bool metgrid_use_sfc
Definition: ERF.H:1238
amrex::Vector< int > num_files_at_level
Definition: ERF.H:796
bool metgrid_debug_quiescent
Definition: ERF.H:1230
bool metgrid_interp_theta
Definition: ERF.H:1235
bool regrid_level_0_on_restart
Definition: ERF.H:1061
int metgrid_force_sfc_k
Definition: ERF.H:1242
void setSubVolVariables(const std::string &pp_subvol_var_names, amrex::Vector< std::string > &subvol_var_names)
Definition: ERF_WriteSubvolume.cpp:8
bool real_extrap_w
Definition: ERF.H:1224
bool metgrid_use_below_sfc
Definition: ERF.H:1237
std::string subvol_file
Definition: ERF.H:1068
amrex::Real metgrid_proximity
Definition: ERF.H:1240
std::string plot2d_file_1
Definition: ERF.H:1066
bool metgrid_debug_dry
Definition: ERF.H:1232
bool metgrid_debug_isothermal
Definition: ERF.H:1231
bool metgrid_debug_psfc
Definition: ERF.H:1233
void ParameterSanityChecks()
Definition: ERF.cpp:2470
bool m_expand_plotvars_to_unif_rr
Definition: ERF.H:1069
std::string check_file
Definition: ERF.H:1089
int metgrid_order
Definition: ERF.H:1241
bool plot_lsm
Definition: ERF.H:1082
void SetModel()
Definition: ERF_LandSurface.H:28
Definition: ERF_MM5.H:26
Definition: ERF_NOAHMP.H:49
Definition: ERF_NullSurf.H:8
Definition: ERF_SLM.H:26
void display(int max_level, std::string pp_prefix)
Definition: ERF_DataStruct.H:719
void init_params(int max_level, std::string pp_prefix)
Definition: ERF_DataStruct.H:126
Here is the call graph for this function:

◆ refinement_criteria_setup()

void ERF::refinement_criteria_setup ( )
private

Function to define the refinement criteria based on user input

238 {
239  if (max_level > 0)
240  {
241  ParmParse pp(pp_prefix);
242  Vector<std::string> refinement_indicators;
243  pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
244 
245  for (int i=0; i<refinement_indicators.size(); ++i)
246  {
247  std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
248 
249  ParmParse ppr(ref_prefix);
250  RealBox realbox;
251  int lev_for_box;
252 
253  int num_real_lo = ppr.countval("in_box_lo");
254  int num_indx_lo = ppr.countval("in_box_lo_indices");
255  int num_real_hi = ppr.countval("in_box_hi");
256  int num_indx_hi = ppr.countval("in_box_hi_indices");
257 
258  AMREX_ALWAYS_ASSERT(num_real_lo == num_real_hi);
259  AMREX_ALWAYS_ASSERT(num_indx_lo == num_indx_hi);
260 
261  if ( !((num_real_lo >= AMREX_SPACEDIM-1 && num_indx_lo == 0) ||
262  (num_indx_lo >= AMREX_SPACEDIM-1 && num_real_lo == 0) ||
263  (num_indx_lo == 0 && num_real_lo == 0)) )
264  {
265  amrex::Abort("Must only specify box for refinement using real OR index space");
266  }
267 
268  if (num_real_lo > 0) {
269  std::vector<Real> rbox_lo(3), rbox_hi(3);
270  ppr.get("max_level",lev_for_box);
271  if (lev_for_box <= max_level)
272  {
273  if (n_error_buf[0] != IntVect::TheZeroVector()) {
274  amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
275  }
276 
277  const Real* plo = geom[lev_for_box].ProbLo();
278  const Real* phi = geom[lev_for_box].ProbHi();
279 
280  ppr.getarr("in_box_lo",rbox_lo,0,num_real_lo);
281  ppr.getarr("in_box_hi",rbox_hi,0,num_real_hi);
282 
283  if (rbox_lo[0] < plo[0]) rbox_lo[0] = plo[0];
284  if (rbox_lo[1] < plo[1]) rbox_lo[1] = plo[1];
285  if (rbox_hi[0] > phi[0]) rbox_hi[0] = phi[0];
286  if (rbox_hi[1] > phi[1]) rbox_hi[1] = phi[1];
287  if (num_real_lo < AMREX_SPACEDIM) {
288  rbox_lo[2] = plo[2];
289  rbox_hi[2] = phi[2];
290  }
291 
292  realbox = RealBox(&(rbox_lo[0]),&(rbox_hi[0]));
293 
294  Print() << "Realbox read in and intersected laterally with domain is " << realbox << std::endl;
295 
296  num_boxes_at_level[lev_for_box] += 1;
297 
298  int ilo, jlo, klo;
299  int ihi, jhi, khi;
300  const auto* dx = geom[lev_for_box].CellSize();
301  ilo = static_cast<int>((rbox_lo[0] - plo[0])/dx[0]);
302  jlo = static_cast<int>((rbox_lo[1] - plo[1])/dx[1]);
303  ihi = static_cast<int>((rbox_hi[0] - plo[0])/dx[0]-1);
304  jhi = static_cast<int>((rbox_hi[1] - plo[1])/dx[1]-1);
305  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
306  // Search for k indices corresponding to nominal grid
307  // AGL heights
308  const Box& domain = geom[lev_for_box].Domain();
309  klo = domain.smallEnd(2) - 1;
310  khi = domain.smallEnd(2) - 1;
311 
312  if (rbox_lo[2] <= zlevels_stag[lev_for_box][domain.smallEnd(2)])
313  {
314  klo = domain.smallEnd(2);
315  }
316  else
317  {
318  for (int k=domain.smallEnd(2); k<=domain.bigEnd(2)+1; ++k) {
319  if (zlevels_stag[lev_for_box][k] > rbox_lo[2]) {
320  klo = k-1;
321  break;
322  }
323  }
324  }
325  AMREX_ASSERT(klo >= domain.smallEnd(2));
326 
327  if (rbox_hi[2] >= zlevels_stag[lev_for_box][domain.bigEnd(2)+1])
328  {
329  khi = domain.bigEnd(2);
330  }
331  else
332  {
333  for (int k=klo+1; k<=domain.bigEnd(2)+1; ++k) {
334  if (zlevels_stag[lev_for_box][k] > rbox_hi[2]) {
335  khi = k-1;
336  break;
337  }
338  }
339  }
340  AMREX_ASSERT((khi <= domain.bigEnd(2)) && (khi > klo));
341 
342  // Need to update realbox because tagging is based on
343  // the initial _un_deformed grid
344  realbox = RealBox(plo[0]+ ilo *dx[0], plo[1]+ jlo *dx[1], plo[2]+ klo *dx[2],
345  plo[0]+(ihi+1)*dx[0], plo[1]+(jhi+1)*dx[1], plo[2]+(khi+1)*dx[2]);
346  } else {
347  klo = static_cast<int>((rbox_lo[2] - plo[2])/dx[2]);
348  khi = static_cast<int>((rbox_hi[2] - plo[2])/dx[2]-1);
349  }
350 
351  Box bx(IntVect(ilo,jlo,klo),IntVect(ihi,jhi,khi));
352  if ( (ilo%ref_ratio[lev_for_box-1][0] != 0) || ((ihi+1)%ref_ratio[lev_for_box-1][0] != 0) ||
353  (jlo%ref_ratio[lev_for_box-1][1] != 0) || ((jhi+1)%ref_ratio[lev_for_box-1][1] != 0) ||
354  (klo%ref_ratio[lev_for_box-1][2] != 0) || ((khi+1)%ref_ratio[lev_for_box-1][2] != 0) )
355  {
356  amrex::Print() << "Box : " << bx << std::endl;
357  amrex::Print() << "RealBox : " << realbox << std::endl;
358  amrex::Print() << "ilo, ihi+1, jlo, jhi+1, klo, khi+1 by ref_ratio : "
359  << ilo%ref_ratio[lev_for_box-1][0] << " " << (ihi+1)%ref_ratio[lev_for_box-1][0] << " "
360  << jlo%ref_ratio[lev_for_box-1][1] << " " << (jhi+1)%ref_ratio[lev_for_box-1][1] << " "
361  << klo%ref_ratio[lev_for_box-1][2] << " " << (khi+1)%ref_ratio[lev_for_box-1][2] << std::endl;
362  amrex::Error("Fine box is not legit with this ref_ratio");
363  }
364  boxes_at_level[lev_for_box].push_back(bx);
365  Print() << "Saving in 'boxes at level' as " << bx << std::endl;
366  } // lev
367 
368  if (solverChoice.init_type == InitType::WRFInput) {
369  if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
370  amrex::Error("Number of boxes doesn't match number of input files");
371 
372  }
373  }
374 
375  } else if (num_indx_lo > 0) {
376 
377  std::vector<int> box_lo(3), box_hi(3);
378  ppr.get("max_level",lev_for_box);
379  if (lev_for_box <= max_level)
380  {
381  if (n_error_buf[0] != IntVect::TheZeroVector()) {
382  amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
383  }
384 
385  ppr.getarr("in_box_lo_indices",box_lo,0,AMREX_SPACEDIM);
386  ppr.getarr("in_box_hi_indices",box_hi,0,AMREX_SPACEDIM);
387 
388  Box bx(IntVect(box_lo[0],box_lo[1],box_lo[2]),IntVect(box_hi[0],box_hi[1],box_hi[2]));
389  amrex::Print() << "BOX " << bx << std::endl;
390 
391  const auto* dx = geom[lev_for_box].CellSize();
392  const Real* plo = geom[lev_for_box].ProbLo();
393  realbox = RealBox(plo[0]+ box_lo[0] *dx[0], plo[1]+ box_lo[1] *dx[1], plo[2]+ box_lo[2] *dx[2],
394  plo[0]+(box_hi[0]+1)*dx[0], plo[1]+(box_hi[1]+1)*dx[1], plo[2]+(box_hi[2]+1)*dx[2]);
395 
396  Print() << "Reading " << bx << " at level " << lev_for_box << std::endl;
397  num_boxes_at_level[lev_for_box] += 1;
398 
399  if ( (box_lo[0]%ref_ratio[lev_for_box-1][0] != 0) || ((box_hi[0]+1)%ref_ratio[lev_for_box-1][0] != 0) ||
400  (box_lo[1]%ref_ratio[lev_for_box-1][1] != 0) || ((box_hi[1]+1)%ref_ratio[lev_for_box-1][1] != 0) ||
401  (box_lo[2]%ref_ratio[lev_for_box-1][2] != 0) || ((box_hi[2]+1)%ref_ratio[lev_for_box-1][2] != 0) )
402  amrex::Error("Fine box is not legit with this ref_ratio");
403  boxes_at_level[lev_for_box].push_back(bx);
404  Print() << "Saving in 'boxes at level' as " << bx << std::endl;
405  } // lev
406 
407  if (solverChoice.init_type == InitType::WRFInput) {
408  if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
409  amrex::Error("Number of boxes doesn't match number of input files");
410 
411  }
412  }
413  }
414 
415  AMRErrorTagInfo info;
416 
417  if (realbox.ok()) {
418  info.SetRealBox(realbox);
419  }
420  if (ppr.countval("start_time") > 0) {
421  Real ref_min_time; ppr.get("start_time",ref_min_time);
422  info.SetMinTime(ref_min_time);
423  }
424  if (ppr.countval("end_time") > 0) {
425  Real ref_max_time; ppr.get("end_time",ref_max_time);
426  info.SetMaxTime(ref_max_time);
427  }
428  if (ppr.countval("max_level") > 0) {
429  int ref_max_level; ppr.get("max_level",ref_max_level);
430  info.SetMaxLevel(ref_max_level);
431  }
432 
433  if (ppr.countval("value_greater")) {
434  int num_val = ppr.countval("value_greater");
435  Vector<Real> value(num_val);
436  ppr.getarr("value_greater",value,0,num_val);
437  std::string field; ppr.get("field_name",field);
438  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GREATER,field,info));
439  }
440  else if (ppr.countval("value_less")) {
441  int num_val = ppr.countval("value_less");
442  Vector<Real> value(num_val);
443  ppr.getarr("value_less",value,0,num_val);
444  std::string field; ppr.get("field_name",field);
445  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::LESS,field,info));
446  }
447  else if (ppr.countval("adjacent_difference_greater")) {
448  int num_val = ppr.countval("adjacent_difference_greater");
449  Vector<Real> value(num_val);
450  ppr.getarr("adjacent_difference_greater",value,0,num_val);
451  std::string field; ppr.get("field_name",field);
452  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GRAD,field,info));
453  }
454  else if (realbox.ok())
455  {
456  ref_tags.push_back(AMRErrorTag(info));
457  } else {
458  Abort(std::string("Unrecognized refinement indicator for " + refinement_indicators[i]).c_str());
459  }
460  } // loop over criteria
461  } // if max_level > 0
462 }
Here is the call graph for this function:

◆ remake_zphys()

void ERF::remake_zphys ( int  lev,
amrex::Real  time,
std::unique_ptr< amrex::MultiFab > &  temp_zphys_nd 
)
675 {
676  if (lev > 0)
677  {
678  //
679  // First interpolate from coarser level
680  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
681  // have been pre-filled - this includes ghost cells both inside and outside
682  // the domain
683  //
684  InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
685  IntVect(0,0,0), // do not fill ghost cells outside the domain
686  *z_phys_nd[lev-1], 0, 0, 1,
687  geom[lev-1], geom[lev],
688  refRatio(lev-1), &node_bilinear_interp,
690 
691  // This recomputes the fine values using the bottom terrain at the fine resolution,
692  // and also fills values of z_phys_nd outside the domain
693  make_terrain_fitted_coords(lev,geom[lev],*temp_zphys_nd,zlevels_stag[lev],phys_bc_type);
694 
695  std::swap(temp_zphys_nd, z_phys_nd[lev]);
696 
697  } // lev > 0
698 
699  if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
700  //
701  // This assumes we have already remade the EBGeometry
702  //
703  terrain_blanking[lev]->setVal(1.0);
704  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, z_phys_nd[lev]->nGrowVect());
705  }
706 
707  // Compute the min dz and pass to the micro model
708  Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
709  micro->Set_dzmin(lev, dzmin);
710 }
Here is the call graph for this function:

◆ RemakeLevel()

void ERF::RemakeLevel ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
483 {
484  if (verbose) {
485  amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl;
486  }
487 
488  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::MovingFittedMesh);
489 
490  BoxArray ba_old(vars_new[lev][Vars::cons].boxArray());
491  DistributionMapping dm_old(vars_new[lev][Vars::cons].DistributionMap());
492 
493  if (verbose) {
494  amrex::Print() <<" OLD BA AT LEVEL " << lev << " " << ba_old << std::endl;
495  }
496 
497  //
498  // Re-define subdomain at this level within the domain such that
499  // 1) all boxes in a given subdomain are "connected"
500  // 2) no boxes in a subdomain touch any boxes in any other subdomain
501  //
502  if (solverChoice.anelastic[lev] == 1) {
503  make_subdomains(ba.simplified_list(), subdomains[lev]);
504  }
505 
506  int ncomp_cons = vars_new[lev][Vars::cons].nComp();
507  IntVect ngrow_state = vars_new[lev][Vars::cons].nGrowVect();
508 
509  int ngrow_vels = ComputeGhostCells(solverChoice);
510 
511  Vector<MultiFab> temp_lev_new(Vars::NumTypes);
512  Vector<MultiFab> temp_lev_old(Vars::NumTypes);
513  MultiFab temp_base_state;
514 
515  std::unique_ptr<MultiFab> temp_zphys_nd;
516 
517  //********************************************************************************************
518  // This allocates all kinds of things, including but not limited to: solution arrays,
519  // terrain arrays and metrics, and base state.
520  // *******************************************************************************************
521  init_stuff(lev, ba, dm, temp_lev_new, temp_lev_old, temp_base_state, temp_zphys_nd);
522 
523  // ********************************************************************************************
524  // Build the data structures for terrain-related quantities
525  // ********************************************************************************************
526  if ( solverChoice.terrain_type == TerrainType::EB ||
527  solverChoice.terrain_type == TerrainType::ImmersedForcing)
528  {
529  const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
530  const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
531  if (solverChoice.terrain_type == TerrainType::EB) {
532  eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
533  } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
534  eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
535  }
536  }
537  remake_zphys(lev, time, temp_zphys_nd);
539 
540  // ********************************************************************************************
541  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
542  // ********************************************************************************************
543  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
544  for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
545  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
546  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
547  }
548  }
549 
550  // ********************************************************************************************
551  // Build the data structures for canopy model (depends upon z_phys)
552  // ********************************************************************************************
554  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
555  }
556 
557  // *****************************************************************************************************
558  // Create the physbcs objects (after initializing the terrain but before calling FillCoarsePatch
559  // *****************************************************************************************************
560  make_physbcs(lev);
561 
562  // ********************************************************************************************
563  // Update the base state at this level by interpolation from coarser level AND copy
564  // from previous (pre-regrid) base_state array
565  // ********************************************************************************************
566  if (lev > 0) {
567  Interpolater* mapper = &cell_cons_interp;
568 
569  Vector<MultiFab*> fmf = {&base_state[lev ], &base_state[lev ]};
570  Vector<MultiFab*> cmf = {&base_state[lev-1], &base_state[lev-1]};
571  Vector<Real> ftime = {time, time};
572  Vector<Real> ctime = {time, time};
573 
574  // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
575  FillPatchTwoLevels(temp_base_state, temp_base_state.nGrowVect(), IntVect(0,0,0),
576  time, cmf, ctime, fmf, ftime,
577  0, 0, temp_base_state.nComp(), geom[lev-1], geom[lev],
578  refRatio(lev-1), mapper, domain_bcs_type,
580 
581  // Impose bc's outside the domain
582  (*physbcs_base[lev])(temp_base_state,0,temp_base_state.nComp(),base_state[lev].nGrowVect());
583 
584  // *************************************************************************************************
585  // This will fill the temporary MultiFabs with data from vars_new
586  // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
587  // NOTE: we must create the new base state before calling FillPatch because we will
588  // interpolate perturbational quantities
589  // *************************************************************************************************
590  FillPatchFineLevel(lev, time, {&temp_lev_new[Vars::cons],&temp_lev_new[Vars::xvel],
591  &temp_lev_new[Vars::yvel],&temp_lev_new[Vars::zvel]},
592  {&temp_lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
593  base_state[lev], temp_base_state, false);
594  } else {
595  temp_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
596  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
597  temp_lev_new[Vars::cons].ParallelCopy(vars_new[lev][Vars::cons],0,0,ncomp_cons,ngrow_state,ngrow_state);
598  temp_lev_new[Vars::xvel].ParallelCopy(vars_new[lev][Vars::xvel],0,0, 1,ngrow_vels,ngrow_vels);
599  temp_lev_new[Vars::yvel].ParallelCopy(vars_new[lev][Vars::yvel],0,0, 1,ngrow_vels,ngrow_vels);
600 
601  temp_lev_new[Vars::zvel].setVal(0.);
602  temp_lev_new[Vars::zvel].ParallelCopy(vars_new[lev][Vars::zvel],0,0, 1,
603  IntVect(ngrow_vels,ngrow_vels,0),IntVect(ngrow_vels,ngrow_vels,0));
604  }
605 
606  // Now swap the pointers since we needed both old and new in the FillPatch
607  std::swap(temp_base_state, base_state[lev]);
608 
609  // ********************************************************************************************
610  // Copy from new into old just in case
611  // ********************************************************************************************
612  MultiFab::Copy(temp_lev_old[Vars::cons],temp_lev_new[Vars::cons],0,0,ncomp_cons,ngrow_state);
613  MultiFab::Copy(temp_lev_old[Vars::xvel],temp_lev_new[Vars::xvel],0,0, 1,ngrow_vels);
614  MultiFab::Copy(temp_lev_old[Vars::yvel],temp_lev_new[Vars::yvel],0,0, 1,ngrow_vels);
615  MultiFab::Copy(temp_lev_old[Vars::zvel],temp_lev_new[Vars::zvel],0,0, 1,IntVect(ngrow_vels,ngrow_vels,0));
616 
617  // ********************************************************************************************
618  // Now swap the pointers
619  // ********************************************************************************************
620  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
621  std::swap(temp_lev_new[var_idx], vars_new[lev][var_idx]);
622  std::swap(temp_lev_old[var_idx], vars_old[lev][var_idx]);
623  }
624 
625  t_new[lev] = time;
626  t_old[lev] = time - 1.e200;
627 
628  // ********************************************************************************************
629  // Build the data structures for calculating diffusive/turbulent terms
630  // ********************************************************************************************
631  update_diffusive_arrays(lev, ba, dm);
632 
633  //********************************************************************************************
634  // Microphysics
635  // *******************************************************************************************
636  int q_size = micro->Get_Qmoist_Size(lev);
637  qmoist[lev].resize(q_size);
638  micro->Define(lev, solverChoice);
639  if (solverChoice.moisture_type != MoistureType::None)
640  {
641  micro->Init(lev, vars_new[lev][Vars::cons],
642  grids[lev], Geom(lev), 0.0,
643  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
644  }
645  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
646  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
647  }
648 
649  //********************************************************************************************
650  // Radiation
651  // *******************************************************************************************
652  if (solverChoice.rad_type != RadiationType::None)
653  {
654  rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
655  }
656 
657  // ********************************************************************************************
658  // Initialize the integrator class
659  // ********************************************************************************************
661 
662  // We need to re-define the FillPatcher if the grids have changed
663  if (lev > 0 && cf_width >= 0) {
664  bool ba_changed = (ba != ba_old);
665  bool dm_changed = (dm != dm_old);
666  if (ba_changed || dm_changed) {
668  }
669  }
670 
671  // ********************************************************************************************
672  // Update the SurfaceLayer arrays at this level
673  // ********************************************************************************************
674  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
675  int nlevs = finest_level+1;
676  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
677  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
678  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,nlevs,
679  mfv_old, Theta_prim[lev], Qv_prim[lev],
680  Qr_prim[lev], z_phys_nd[lev],
681  Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
683  sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
684  }
685 
686  // These calls are done in AmrCore::regrid if this is a regrid at lev > 0
687  // For a level 0 regrid we must explicitly do them here
688  if (lev == 0) {
689  // Define grids[lev] to be ba
690  SetBoxArray(lev, ba);
691 
692  // Define dmap[lev] to be dm
693  SetDistributionMap(lev, dm);
694  }
695 
696 #ifdef ERF_USE_PARTICLES
697  particleData.Redistribute();
698 #endif
699 }
void remake_zphys(int lev, amrex::Real time, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)
Definition: ERF_MakeNewArrays.cpp:674

◆ restart()

void ERF::restart ( )
1870 {
1871  auto dRestartTime0 = amrex::second();
1872 
1874 
1876  //
1877  // Coarsening before we split the grids ensures that each resulting
1878  // grid will have an even number of cells in each direction.
1879  //
1880  BoxArray new_ba(amrex::coarsen(Geom(0).Domain(),2));
1881  //
1882  // Now split up into list of grids within max_grid_size[0] limit.
1883  //
1884  new_ba.maxSize(max_grid_size[0]/2);
1885  //
1886  // Now refine these boxes back to level 0.
1887  //
1888  new_ba.refine(2);
1889 
1890  if (refine_grid_layout) {
1891  ChopGrids(0, new_ba, ParallelDescriptor::NProcs());
1892  }
1893 
1894  if (new_ba != grids[0]) {
1895  DistributionMapping new_dm(new_ba);
1896  RemakeLevel(0,t_new[0],new_ba,new_dm);
1897  }
1898  }
1899 
1900 #ifdef ERF_USE_PARTICLES
1901  // We call this here without knowing whether the particles have already been initialized or not
1902  initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,t_new[0]);
1903 #endif
1904 
1905  Real cur_time = t_new[0];
1906  if (m_check_per > 0.) {last_check_file_time = cur_time;}
1907  if (m_plot2d_per_1 > 0.) {last_plot2d_file_time_1 = std::floor(cur_time/m_plot2d_per_1) * m_plot2d_per_1;}
1908  if (m_plot2d_per_2 > 0.) {last_plot2d_file_time_2 = std::floor(cur_time/m_plot2d_per_2) * m_plot2d_per_2;}
1909  if (m_plot3d_per_1 > 0.) {last_plot3d_file_time_1 = std::floor(cur_time/m_plot3d_per_1) * m_plot3d_per_1;}
1910  if (m_plot3d_per_2 > 0.) {last_plot3d_file_time_2 = std::floor(cur_time/m_plot3d_per_2) * m_plot3d_per_2;}
1911 
1912  if (m_check_int > 0.) {last_check_file_step = istep[0];}
1913  if (m_plot2d_int_1 > 0.) {last_plot2d_file_step_1 = istep[0];}
1914  if (m_plot2d_int_2 > 0.) {last_plot2d_file_step_2 = istep[0];}
1915  if (m_plot3d_int_1 > 0.) {last_plot3d_file_step_1 = istep[0];}
1916  if (m_plot3d_int_2 > 0.) {last_plot3d_file_step_2 = istep[0];}
1917 
1918  if (verbose > 0)
1919  {
1920  auto dRestartTime = amrex::second() - dRestartTime0;
1921  ParallelDescriptor::ReduceRealMax(dRestartTime,ParallelDescriptor::IOProcessorNumber());
1922  amrex::Print() << "Restart time = " << dRestartTime << " seconds." << '\n';
1923  }
1924 }
void RemakeLevel(int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
Definition: ERF_MakeNewLevel.cpp:482
void ReadCheckpointFile()
Definition: ERF_Checkpoint.cpp:447

◆ sample_lines()

void ERF::sample_lines ( int  lev,
amrex::Real  time,
amrex::IntVect  cell,
amrex::MultiFab &  mf 
)

Utility function for sampling data along a line along the z-dimension at the (x,y) indices specified and writes it to an output file.

Parameters
levCurrent level
timeCurrent time
cellIntVect containing the x,y-dimension indices to sample along z
mfMultiFab from which we sample the data
564 {
565  int ifile = 0;
566 
567  const int ncomp = mf.nComp(); // cell-centered state vars
568 
569  MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
570  average_face_to_cellcenter(mf_vels, 0,
571  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
572 
573  //
574  // Sample the data at a line (in direction "dir") in space
575  // In this case we sample in the vertical direction so dir = 2
576  // The "k" value of "cell" is ignored
577  //
578  int dir = 2;
579  MultiFab my_line = get_line_data(mf, dir, cell);
580  MultiFab my_line_vels = get_line_data(mf_vels, dir, cell);
581  MultiFab my_line_tau11 = get_line_data(*Tau[lev][TauType::tau11], dir, cell);
582  MultiFab my_line_tau12 = get_line_data(*Tau[lev][TauType::tau12], dir, cell);
583  MultiFab my_line_tau13 = get_line_data(*Tau[lev][TauType::tau13], dir, cell);
584  MultiFab my_line_tau22 = get_line_data(*Tau[lev][TauType::tau22], dir, cell);
585  MultiFab my_line_tau23 = get_line_data(*Tau[lev][TauType::tau23], dir, cell);
586  MultiFab my_line_tau33 = get_line_data(*Tau[lev][TauType::tau33], dir, cell);
587 
588  for (MFIter mfi(my_line, false); mfi.isValid(); ++mfi)
589  {
590  // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
591 
592  std::ostream& sample_log = SampleLineLog(ifile);
593  if (sample_log.good()) {
594  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << time;
595  const auto& my_line_arr = my_line[0].const_array();
596  const auto& my_line_vels_arr = my_line_vels[0].const_array();
597  const auto& my_line_tau11_arr = my_line_tau11[0].const_array();
598  const auto& my_line_tau12_arr = my_line_tau12[0].const_array();
599  const auto& my_line_tau13_arr = my_line_tau13[0].const_array();
600  const auto& my_line_tau22_arr = my_line_tau22[0].const_array();
601  const auto& my_line_tau23_arr = my_line_tau23[0].const_array();
602  const auto& my_line_tau33_arr = my_line_tau33[0].const_array();
603  const Box& my_box = my_line[0].box();
604  const int klo = my_box.smallEnd(2);
605  const int khi = my_box.bigEnd(2);
606  int i = cell[0];
607  int j = cell[1];
608  for (int n = 0; n < ncomp; n++) {
609  for (int k = klo; k <= khi; k++) {
610  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_arr(i,j,k,n);
611  }
612  }
613  for (int n = 0; n < AMREX_SPACEDIM; n++) {
614  for (int k = klo; k <= khi; k++) {
615  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_vels_arr(i,j,k,n);
616  }
617  }
618  for (int k = klo; k <= khi; k++) {
619  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau11_arr(i,j,k);
620  }
621  for (int k = klo; k <= khi; k++) {
622  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau12_arr(i,j,k);
623  }
624  for (int k = klo; k <= khi; k++) {
625  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau13_arr(i,j,k);
626  }
627  for (int k = klo; k <= khi; k++) {
628  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau22_arr(i,j,k);
629  }
630  for (int k = klo; k <= khi; k++) {
631  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau23_arr(i,j,k);
632  }
633  for (int k = klo; k <= khi; k++) {
634  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau33_arr(i,j,k);
635  }
636  sample_log << std::endl;
637  } // if good
638  } // mfi
639 }
const int datwidth
Definition: ERF.H:1021
AMREX_FORCE_INLINE std::ostream & SampleLineLog(int i)
Definition: ERF.H:1457
const int datprecision
Definition: ERF.H:1022

◆ sample_points()

void ERF::sample_points ( int  lev,
amrex::Real  time,
amrex::IntVect  cell,
amrex::MultiFab &  mf 
)

Utility function for sampling MultiFab data at a specified cell index.

Parameters
levLevel for the associated MultiFab data
timeCurrent time
cellIntVect containing the indexes for the cell where we want to sample
mfMultiFab from which we wish to sample data
528 {
529  int ifile = 0;
530 
531  //
532  // Sample the data at a single point in space
533  //
534  int ncomp = mf.nComp();
535  Vector<Real> my_point = get_cell_data(mf, cell);
536 
537  if (!my_point.empty()) {
538 
539  // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
540 
541  std::ostream& sample_log = SamplePointLog(ifile);
542  if (sample_log.good()) {
543  sample_log << std::setw(datwidth) << time;
544  for (int i = 0; i < ncomp; ++i)
545  {
546  sample_log << std::setw(datwidth) << my_point[i];
547  }
548  sample_log << std::endl;
549  } // if good
550  } // only write from processor that holds the cell
551 }
AMREX_FORCE_INLINE std::ostream & SamplePointLog(int i)
Definition: ERF.H:1443

◆ SampleLine()

amrex::IntVect& ERF::SampleLine ( int  i)
inlineprivate
1484  {
1485  return sampleline[i];
1486  }

◆ SampleLineLog()

AMREX_FORCE_INLINE std::ostream& ERF::SampleLineLog ( int  i)
inlineprivate
1458  {
1459  return *samplelinelog[i];
1460  }

◆ SampleLineLogName()

std::string ERF::SampleLineLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith samplelinelog file.

1617 { return samplelinelogname[i]; }

◆ SamplePoint()

amrex::IntVect& ERF::SamplePoint ( int  i)
inlineprivate
1471  {
1472  return samplepoint[i];
1473  }

◆ SamplePointLog()

AMREX_FORCE_INLINE std::ostream& ERF::SamplePointLog ( int  i)
inlineprivate
1444  {
1445  return *sampleptlog[i];
1446  }

◆ SamplePointLogName()

std::string ERF::SamplePointLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith sampleptlog file.

1614 { return sampleptlogname[i]; }

◆ setPlotVariables()

void ERF::setPlotVariables ( const std::string &  pp_plot_var_names,
amrex::Vector< std::string > &  plot_var_names 
)
private
25 {
26  ParmParse pp(pp_prefix);
27 
28  if (pp.contains(pp_plot_var_names.c_str()))
29  {
30  std::string nm;
31 
32  int nPltVars = pp.countval(pp_plot_var_names.c_str());
33 
34  for (int i = 0; i < nPltVars; i++)
35  {
36  pp.get(pp_plot_var_names.c_str(), nm, i);
37 
38  // Add the named variable to our list of plot variables
39  // if it is not already in the list
40  if (!containerHasElement(plot_var_names, nm)) {
41  plot_var_names.push_back(nm);
42  }
43  }
44  } else {
45  //
46  // The default is to add none of the variables to the list
47  //
48  plot_var_names.clear();
49  }
50 
51  // Get state variables in the same order as we define them,
52  // since they may be in any order in the input list
53  Vector<std::string> tmp_plot_names;
54 
55  for (int i = 0; i < cons_names.size(); ++i) {
56  if ( containerHasElement(plot_var_names, cons_names[i]) ) {
57  if (solverChoice.moisture_type == MoistureType::None) {
58  if (cons_names[i] != "rhoQ1" && cons_names[i] != "rhoQ2" && cons_names[i] != "rhoQ3" &&
59  cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
60  {
61  tmp_plot_names.push_back(cons_names[i]);
62  }
63  } else if (solverChoice.moisture_type == MoistureType::Kessler) { // allow rhoQ1, rhoQ2, rhoQ3
64  if (cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
65  {
66  tmp_plot_names.push_back(cons_names[i]);
67  }
68  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
69  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
70  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow rhoQ1, rhoQ2
71  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ4" &&
72  cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
73  {
74  tmp_plot_names.push_back(cons_names[i]);
75  }
76  } else if ( (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
77  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow rhoQ1, rhoQ2, rhoQ4
78  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
79  {
80  tmp_plot_names.push_back(cons_names[i]);
81  }
82  } else
83  {
84  // For moisture_type SAM and Morrison we have all six variables
85  tmp_plot_names.push_back(cons_names[i]);
86  }
87  }
88  }
89 
90  // check for velocity since it's not in cons_names
91  // if we are asked for any velocity component, we will need them all
92  if (containerHasElement(plot_var_names, "x_velocity") ||
93  containerHasElement(plot_var_names, "y_velocity") ||
94  containerHasElement(plot_var_names, "z_velocity")) {
95  tmp_plot_names.push_back("x_velocity");
96  tmp_plot_names.push_back("y_velocity");
97  tmp_plot_names.push_back("z_velocity");
98  }
99 
100  //
101  // If the model we are running doesn't have the variable listed in the inputs file,
102  // just ignore it rather than aborting
103  //
104  for (int i = 0; i < derived_names.size(); ++i) {
105  if ( containerHasElement(plot_var_names, derived_names[i]) ) {
106  bool ok_to_add = ( (solverChoice.terrain_type == TerrainType::ImmersedForcing) ||
107  (derived_names[i] != "terrain_IB_mask") );
108  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
109  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
110  (derived_names[i] != "detJ") );
111  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
112  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
113  (derived_names[i] != "z_phys") );
114 #ifndef ERF_USE_WINDFARM
115  ok_to_add &= (derived_names[i] != "SMark0" && derived_names[i] != "SMark1");
116 #endif
117  if (ok_to_add)
118  {
119  if (solverChoice.moisture_type == MoistureType::None) { // no moist quantities allowed
120  if (derived_names[i] != "qv" && derived_names[i] != "qc" && derived_names[i] != "qrain" &&
121  derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
122  derived_names[i] != "qt" && derived_names[i] != "qn" && derived_names[i] != "qp" &&
123  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
124  {
125  tmp_plot_names.push_back(derived_names[i]);
126  }
127  } else if ( (solverChoice.moisture_type == MoistureType::Kessler ) ||
128  (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
129  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow qv, qc, qrain
130  if (derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
131  derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
132  {
133  tmp_plot_names.push_back(derived_names[i]);
134  }
135  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
136  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
137  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow qv, qc
138  if (derived_names[i] != "qrain" &&
139  derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
140  derived_names[i] != "qp" &&
141  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
142  {
143  tmp_plot_names.push_back(derived_names[i]);
144  }
145  } else
146  {
147  // For moisture_type SAM and Morrison we have all moist quantities
148  tmp_plot_names.push_back(derived_names[i]);
149  }
150  } // use_terrain?
151  } // hasElement
152  }
153 
154 #ifdef ERF_USE_WINDFARM
155  for (int i = 0; i < derived_names.size(); ++i) {
156  if ( containerHasElement(plot_var_names, derived_names[i]) ) {
157  if(solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP) {
158  if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0") {
159  tmp_plot_names.push_back(derived_names[i]);
160  }
161  }
162  if( solverChoice.windfarm_type == WindFarmType::SimpleAD or
163  solverChoice.windfarm_type == WindFarmType::GeneralAD ) {
164  if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0" or derived_names[i] == "SMark1") {
165  tmp_plot_names.push_back(derived_names[i]);
166  }
167  }
168  }
169  }
170 #endif
171 
172 #ifdef ERF_USE_PARTICLES
173  const auto& particles_namelist( particleData.getNamesUnalloc() );
174  for (auto it = particles_namelist.cbegin(); it != particles_namelist.cend(); ++it) {
175  std::string tmp( (*it)+"_count" );
176  if (containerHasElement(plot_var_names, tmp) ) {
177  tmp_plot_names.push_back(tmp);
178  }
179  }
180 #endif
181 
182  plot_var_names = tmp_plot_names;
183 }
const amrex::Vector< std::string > derived_names
Definition: ERF.H:1106
const amrex::Vector< std::string > cons_names
Definition: ERF.H:1099
Here is the call graph for this function:

◆ setPlotVariables2D()

void ERF::setPlotVariables2D ( const std::string &  pp_plot_var_names,
amrex::Vector< std::string > &  plot_var_names 
)
private
187 {
188  ParmParse pp(pp_prefix);
189 
190  if (pp.contains(pp_plot_var_names.c_str()))
191  {
192  std::string nm;
193 
194  int nPltVars = pp.countval(pp_plot_var_names.c_str());
195 
196  for (int i = 0; i < nPltVars; i++)
197  {
198  pp.get(pp_plot_var_names.c_str(), nm, i);
199 
200  // Add the named variable to our list of plot variables
201  // if it is not already in the list
202  if (!containerHasElement(plot_var_names, nm)) {
203  plot_var_names.push_back(nm);
204  }
205  }
206  } else {
207  //
208  // The default is to add none of the variables to the list
209  //
210  plot_var_names.clear();
211  }
212 
213  // Get state variables in the same order as we define them,
214  // since they may be in any order in the input list
215  Vector<std::string> tmp_plot_names;
216 
217  // 2D plot variables
218  for (int i = 0; i < derived_names_2d.size(); ++i) {
219  if (containerHasElement(plot_var_names, derived_names_2d[i]) ) {
220  tmp_plot_names.push_back(derived_names_2d[i]);
221  }
222  }
223 
224  plot_var_names = tmp_plot_names;
225 }
const amrex::Vector< std::string > derived_names_2d
Definition: ERF.H:1147
Here is the call graph for this function:

◆ setRayleighRefFromSounding()

void ERF::setRayleighRefFromSounding ( bool  restarting)
private

Set Rayleigh mean profiles from input sounding.

Sets the Rayleigh Damping averaged quantities from an externally supplied input sounding data file.

Parameters
[in]restartingBoolean parameter that indicates whether we are currently restarting from a checkpoint file.
95 {
96  // If we are restarting then we haven't read the input_sounding file yet
97  // so we need to read it here
98  // TODO: should we store this information in the checkpoint file instead?
99  if (restarting) {
101  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
103  }
104  }
105 
106  const Real* z_inp_sound = input_sounding_data.z_inp_sound[0].dataPtr();
107  const Real* U_inp_sound = input_sounding_data.U_inp_sound[0].dataPtr();
108  const Real* V_inp_sound = input_sounding_data.V_inp_sound[0].dataPtr();
109  const Real* theta_inp_sound = input_sounding_data.theta_inp_sound[0].dataPtr();
110  const int inp_sound_size = input_sounding_data.size(0);
111 
112  int refine_fac{1};
113  for (int lev = 0; lev <= finest_level; lev++)
114  {
115  const int klo = geom[lev].Domain().smallEnd(2);
116  const int khi = geom[lev].Domain().bigEnd(2);
117  const int Nz = khi - klo + 1;
118 
119  Vector<Real> zcc(Nz);
120  Vector<Real> zlevels_sub(zlevels_stag[0].begin()+klo/refine_fac,
121  zlevels_stag[0].begin()+khi/refine_fac+2);
122  expand_and_interpolate_1d(zcc, zlevels_sub, refine_fac, true);
123 #if 0
124  amrex::AllPrint() << "lev="<<lev<<" : (refine_fac="<<refine_fac<<",klo="<<klo<<",khi="<<khi<<") ";
125  for (int k = 0; k < zlevels_sub.size(); k++) { amrex::AllPrint() << zlevels_sub[k] << " "; }
126  amrex::AllPrint() << " --> ";
127  for (int k = 0; k < Nz; k++) { amrex::AllPrint() << zcc[k] << " "; }
128  amrex::AllPrint() << std::endl;
129 #endif
130 
131  for (int k = 0; k < Nz; k++)
132  {
133  h_rayleigh_ptrs[lev][Rayleigh::ubar][k] = interpolate_1d(z_inp_sound, U_inp_sound, zcc[k], inp_sound_size);
134  h_rayleigh_ptrs[lev][Rayleigh::vbar][k] = interpolate_1d(z_inp_sound, V_inp_sound, zcc[k], inp_sound_size);
135  h_rayleigh_ptrs[lev][Rayleigh::wbar][k] = Real(0.0);
136  h_rayleigh_ptrs[lev][Rayleigh::thetabar][k] = interpolate_1d(z_inp_sound, theta_inp_sound, zcc[k], inp_sound_size);
137  }
138 
139  // Copy from host version to device version
140  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::ubar].begin(), h_rayleigh_ptrs[lev][Rayleigh::ubar].end(),
141  d_rayleigh_ptrs[lev][Rayleigh::ubar].begin());
142  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::vbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::vbar].end(),
143  d_rayleigh_ptrs[lev][Rayleigh::vbar].begin());
144  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::wbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::wbar].end(),
145  d_rayleigh_ptrs[lev][Rayleigh::wbar].begin());
146  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::thetabar].begin(), h_rayleigh_ptrs[lev][Rayleigh::thetabar].end(),
147  d_rayleigh_ptrs[lev][Rayleigh::thetabar].begin());
148 
149  refine_fac *= ref_ratio[lev][2];
150  }
151 }
AMREX_FORCE_INLINE void expand_and_interpolate_1d(amrex::Vector< amrex::Real > &znew, const amrex::Vector< amrex::Real > &zorig, int refine_fac, bool destag=false)
Definition: ERF_Interpolation_1D.H:85
amrex::Vector< amrex::Vector< amrex::Real > > theta_inp_sound
Definition: ERF_InputSoundingData.H:404
amrex::Vector< amrex::Vector< amrex::Real > > z_inp_sound
Definition: ERF_InputSoundingData.H:404
amrex::Vector< amrex::Vector< amrex::Real > > U_inp_sound
Definition: ERF_InputSoundingData.H:404
amrex::Vector< amrex::Vector< amrex::Real > > V_inp_sound
Definition: ERF_InputSoundingData.H:404
int size(int itime) const
Definition: ERF_InputSoundingData.H:379
Here is the call graph for this function:

◆ setRecordDataInfo()

void ERF::setRecordDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1514  {
1515  if (amrex::ParallelDescriptor::IOProcessor())
1516  {
1517  datalog[i] = std::make_unique<std::fstream>();
1518  datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1519  if (!datalog[i]->good()) {
1520  amrex::FileOpenFailed(filename);
1521  }
1522  }
1523  amrex::ParallelDescriptor::Barrier("ERF::setRecordDataInfo");
1524  }

◆ setRecordDerDataInfo()

void ERF::setRecordDerDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1527  {
1528  if (amrex::ParallelDescriptor::IOProcessor())
1529  {
1530  der_datalog[i] = std::make_unique<std::fstream>();
1531  der_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1532  if (!der_datalog[i]->good()) {
1533  amrex::FileOpenFailed(filename);
1534  }
1535  }
1536  amrex::ParallelDescriptor::Barrier("ERF::setRecordDerDataInfo");
1537  }

◆ setRecordEnergyDataInfo()

void ERF::setRecordEnergyDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1540  {
1541  if (amrex::ParallelDescriptor::IOProcessor())
1542  {
1543  tot_e_datalog[i] = std::make_unique<std::fstream>();
1544  tot_e_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1545  if (!tot_e_datalog[i]->good()) {
1546  amrex::FileOpenFailed(filename);
1547  }
1548  }
1549  amrex::ParallelDescriptor::Barrier("ERF::setRecordEnergyDataInfo");
1550  }

◆ setRecordSampleLineInfo()

void ERF::setRecordSampleLineInfo ( int  i,
int  lev,
amrex::IntVect &  cell,
const std::string &  filename 
)
inlineprivate
1570  {
1571  amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
1572  for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
1573  {
1574  const amrex::Box& bx = mfi.validbox();
1575  if (bx.contains(cell)) {
1576  samplelinelog[i] = std::make_unique<std::fstream>();
1577  samplelinelog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1578  if (!samplelinelog[i]->good()) {
1579  amrex::FileOpenFailed(filename);
1580  }
1581  }
1582  }
1583  amrex::ParallelDescriptor::Barrier("ERF::setRecordSampleLineInfo");
1584  }

◆ setRecordSamplePointInfo()

void ERF::setRecordSamplePointInfo ( int  i,
int  lev,
amrex::IntVect &  cell,
const std::string &  filename 
)
inlineprivate
1553  {
1554  amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
1555  for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
1556  {
1557  const amrex::Box& bx = mfi.validbox();
1558  if (bx.contains(cell)) {
1559  sampleptlog[i] = std::make_unique<std::fstream>();
1560  sampleptlog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1561  if (!sampleptlog[i]->good()) {
1562  amrex::FileOpenFailed(filename);
1563  }
1564  }
1565  }
1566  amrex::ParallelDescriptor::Barrier("ERF::setRecordSamplePointInfo");
1567  }

◆ setSpongeRefFromSounding()

void ERF::setSpongeRefFromSounding ( bool  restarting)
private

Set sponge mean profiles from input sounding.

Sets the sponge damping averaged quantities from an externally supplied input sponge data file.

Parameters
[in]restartingBoolean parameter that indicates whether we are currently restarting from a checkpoint file.
66 {
67  // If we are restarting then we haven't read the input_sponge file yet
68  // so we need to read it here
69  // TODO: should we store this information in the checkpoint file instead?
70  if (restarting) {
72  }
73 
74  const Real* z_inp_sponge = input_sponge_data.z_inp_sponge.dataPtr();
75  const Real* U_inp_sponge = input_sponge_data.U_inp_sponge.dataPtr();
76  const Real* V_inp_sponge = input_sponge_data.V_inp_sponge.dataPtr();
77  const int inp_sponge_size = input_sponge_data.size();
78 
79  for (int lev = 0; lev <= finest_level; lev++)
80  {
81  const int khi = geom[lev].Domain().bigEnd()[2];
82  Vector<Real> zcc(khi+1);
83 
84  if (z_phys_cc[lev]) {
85  // use_terrain=1
86  // calculate the damping strength based on the max height at each k
88  } else {
89  const auto *const prob_lo = geom[lev].ProbLo();
90  const auto *const dx = geom[lev].CellSize();
91  for (int k = 0; k <= khi; k++)
92  {
93  zcc[k] = prob_lo[2] + (k+0.5) * dx[2];
94  }
95  }
96 
97  for (int k = 0; k <= khi; k++)
98  {
99  h_sponge_ptrs[lev][Sponge::ubar_sponge][k] = interpolate_1d(z_inp_sponge, U_inp_sponge, zcc[k], inp_sponge_size);
100  h_sponge_ptrs[lev][Sponge::vbar_sponge][k] = interpolate_1d(z_inp_sponge, V_inp_sponge, zcc[k], inp_sponge_size);
101  }
102 
103  // Copy from host version to device version
104  Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::ubar_sponge].begin(), h_sponge_ptrs[lev][Sponge::ubar_sponge].end(),
105  d_sponge_ptrs[lev][Sponge::ubar_sponge].begin());
106  Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::vbar_sponge].begin(), h_sponge_ptrs[lev][Sponge::vbar_sponge].end(),
107  d_sponge_ptrs[lev][Sponge::vbar_sponge].begin());
108  }
109 }
AMREX_FORCE_INLINE void reduce_to_max_per_height(amrex::Vector< amrex::Real > &v, std::unique_ptr< amrex::MultiFab > &mf)
Definition: ERF_ParFunctions.H:8
amrex::Vector< amrex::Real > V_inp_sponge
Definition: ERF_InputSpongeData.H:111
amrex::Vector< amrex::Real > z_inp_sponge
Definition: ERF_InputSpongeData.H:111
amrex::Vector< amrex::Real > U_inp_sponge
Definition: ERF_InputSpongeData.H:111
int size() const
Definition: ERF_InputSpongeData.H:99
Here is the call graph for this function:

◆ setSubVolVariables()

void ERF::setSubVolVariables ( const std::string &  pp_subvol_var_names,
amrex::Vector< std::string > &  subvol_var_names 
)
private
10 {
11  ParmParse pp(pp_prefix);
12 
13  std::string nm;
14 
15  int nSubVolVars = pp.countval(pp_subvol_var_names.c_str());
16 
17  // We pre-populate the list with velocities, but allow these to be over-written
18  // by user input
19  if (nSubVolVars == 0)
20  {
21  subvol_var_names.push_back("x_velocity");
22  subvol_var_names.push_back("y_velocity");
23  subvol_var_names.push_back("z_velocity");
24 
25  } else {
26  for (int i = 0; i < nSubVolVars; i++)
27  {
28  pp.get(pp_subvol_var_names.c_str(), nm, i);
29 
30  // Add the named variable to our list of subvol variables
31  // if it is not already in the list
32  if (!containerHasElement(subvol_var_names, nm)) {
33  subvol_var_names.push_back(nm);
34  }
35  }
36  }
37 
38  // Get state variables in the same order as we define them,
39  // since they may be in any order in the input list
40  Vector<std::string> tmp_plot_names;
41 
42  for (int i = 0; i < cons_names.size(); ++i) {
43  if ( containerHasElement(subvol_var_names, cons_names[i]) ) {
44  if (solverChoice.moisture_type == MoistureType::None) {
45  if (cons_names[i] != "rhoQ1" && cons_names[i] != "rhoQ2" && cons_names[i] != "rhoQ3" &&
46  cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
47  {
48  tmp_plot_names.push_back(cons_names[i]);
49  }
50  } else if (solverChoice.moisture_type == MoistureType::Kessler) { // allow rhoQ1, rhoQ2, rhoQ3
51  if (cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
52  {
53  tmp_plot_names.push_back(cons_names[i]);
54  }
55  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
56  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
57  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow rhoQ1, rhoQ2
58  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ4" &&
59  cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
60  {
61  tmp_plot_names.push_back(cons_names[i]);
62  }
63  } else if ( (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
64  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow rhoQ1, rhoQ2, rhoQ4
65  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
66  {
67  tmp_plot_names.push_back(cons_names[i]);
68  }
69  } else
70  {
71  // For moisture_type SAM and Morrison we have all six variables
72  tmp_plot_names.push_back(cons_names[i]);
73  }
74  }
75  }
76 
77  // Check for velocity since it's not in cons_names
78  if (containerHasElement(subvol_var_names, "x_velocity")) {
79  tmp_plot_names.push_back("x_velocity");
80  }
81  if (containerHasElement(subvol_var_names, "y_velocity")) {
82  tmp_plot_names.push_back("y_velocity");
83  }
84  if (containerHasElement(subvol_var_names, "z_velocity")) {
85  tmp_plot_names.push_back("z_velocity");
86  }
87 
88  //
89  // If the model we are running doesn't have the variable listed in the inputs file,
90  // just ignore it rather than aborting
91  //
92  for (int i = 0; i < derived_subvol_names.size(); ++i) {
93  if ( containerHasElement(subvol_var_names, derived_names[i]) ) {
94  bool ok_to_add = ( (solverChoice.terrain_type == TerrainType::ImmersedForcing) ||
95  (derived_names[i] != "terrain_IB_mask") );
96  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
97  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
98  (derived_names[i] != "detJ") );
99  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
100  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
101  (derived_names[i] != "z_phys") );
102  if (ok_to_add)
103  {
104  if (solverChoice.moisture_type == MoistureType::None) { // no moist quantities allowed
105  if (derived_names[i] != "qv" && derived_names[i] != "qc" && derived_names[i] != "qrain" &&
106  derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
107  derived_names[i] != "qt" && derived_names[i] != "qn" && derived_names[i] != "qp" &&
108  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
109  {
110  tmp_plot_names.push_back(derived_names[i]);
111  }
112  } else if ( (solverChoice.moisture_type == MoistureType::Kessler ) ||
113  (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
114  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow qv, qc, qrain
115  if (derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
116  derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
117  {
118  tmp_plot_names.push_back(derived_names[i]);
119  }
120  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
121  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
122  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow qv, qc
123  if (derived_names[i] != "qrain" &&
124  derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
125  derived_names[i] != "qp" &&
126  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
127  {
128  tmp_plot_names.push_back(derived_names[i]);
129  }
130  } else
131  {
132  // For moisture_type SAM and Morrison we have all moist quantities
133  tmp_plot_names.push_back(derived_names[i]);
134  }
135  } // use_terrain?
136  } // hasElement
137  }
138 
139  subvol_var_names = tmp_plot_names;
140 }
const amrex::Vector< std::string > derived_subvol_names
Definition: ERF.H:1157
Here is the call graph for this function:

◆ solve_with_EB_mlmg()

void ERF::solve_with_EB_mlmg ( int  lev,
amrex::Vector< amrex::MultiFab > &  rhs,
amrex::Vector< amrex::MultiFab > &  p,
amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &  fluxes 
)

Solve the Poisson equation using EB_enabled MLMG Note that the level may or may not be level 0.

20 {
21  BL_PROFILE("ERF::solve_with_EB_mlmg()");
22 
23  auto const dom_lo = lbound(geom[lev].Domain());
24  auto const dom_hi = ubound(geom[lev].Domain());
25 
26  LPInfo info;
27  // Allow a hidden direction if the domain is one cell wide in any lateral direction
28  if (dom_lo.x == dom_hi.x) {
29  info.setHiddenDirection(0);
30  } else if (dom_lo.y == dom_hi.y) {
31  info.setHiddenDirection(1);
32  }
33 
34  // Make sure the solver only sees the levels over which we are solving
35  Vector<BoxArray> ba_tmp; ba_tmp.push_back(rhs[0].boxArray());
36  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(rhs[0].DistributionMap());
37  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
38 
39  auto bclo = get_projection_bc(Orientation::low);
40  auto bchi = get_projection_bc(Orientation::high);
41 
42  // amrex::Print() << "BCLO " << bclo[0] << " " << bclo[1] << " " << bclo[2] << std::endl;
43  // amrex::Print() << "BCHI " << bchi[0] << " " << bchi[1] << " " << bchi[2] << std::endl;
44 
47 
48  // ****************************************************************************
49  // Multigrid solve
50  // ****************************************************************************
51 
52  MLEBABecLap mleb (geom_tmp, ba_tmp, dm_tmp, info, {&EBFactory(lev)});
53 
54  mleb.setMaxOrder(2);
55  mleb.setDomainBC(bclo, bchi);
56  mleb.setLevelBC(0, nullptr);
57 
58  //
59  // This sets A = 0, B = 1 so that
60  // the operator A alpha - b del dot beta grad to b
61  // becomes - del dot beta grad
62  //
63  mleb.setScalars(0.0, 1.0);
64 
65  Array<MultiFab,AMREX_SPACEDIM> bcoef;
66  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
67  bcoef[idim].define(convert(ba_tmp[0],IntVect::TheDimensionVector(idim)),
68  dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
69  bcoef[idim].setVal(-1.0);
70  }
71  mleb.setBCoeffs(0, amrex::GetArrOfConstPtrs(bcoef));
72 
73  MLMG mlmg(mleb);
74 
75  int max_iter = 100;
76  mlmg.setMaxIter(max_iter);
77  mlmg.setVerbose(mg_verbose);
78  mlmg.setBottomVerbose(0);
79 
80  mlmg.solve(GetVecOfPtrs(phi), GetVecOfConstPtrs(rhs), reltol, abstol);
81 
82  mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
83 
84  ImposeBCsOnPhi(lev,phi[0], geom[lev].Domain());
85 
86  //
87  // This arises because we solve MINUS del dot beta grad phi = div (rho u)
88  //
89  fluxes[0][0].mult(-1.);
90  fluxes[0][1].mult(-1.);
91  fluxes[0][2].mult(-1.);
92 }
void ImposeBCsOnPhi(int lev, amrex::MultiFab &phi, const amrex::Box &subdomain)
Definition: ERF_ImposeBCsOnPhi.cpp:12

◆ solve_with_gmres()

void ERF::solve_with_gmres ( int  lev,
const amrex::Box &  subdomain,
amrex::MultiFab &  rhs,
amrex::MultiFab &  p,
amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &  fluxes,
amrex::MultiFab &  ax_sub,
amrex::MultiFab &  ay_sub,
amrex::MultiFab &  az_sub,
amrex::MultiFab &  ,
amrex::MultiFab &  znd_sub 
)

Solve the Poisson equation using FFT-preconditioned GMRES

16 {
17 #ifdef ERF_USE_FFT
18  BL_PROFILE("ERF::solve_with_gmres()");
19 
22 
23  auto const dom_lo = lbound(Geom(lev).Domain());
24  auto const dom_hi = ubound(Geom(lev).Domain());
25 
26  auto const sub_lo = lbound(subdomain);
27  auto const sub_hi = ubound(subdomain);
28 
29  auto dx = Geom(lev).CellSizeArray();
30 
31  Geometry my_geom;
32 
33  Array<int,AMREX_SPACEDIM> is_per; is_per[0] = 0; is_per[1] = 0; is_per[2] = 0;
34  if (Geom(lev).isPeriodic(0) && sub_lo.x == dom_lo.x && sub_hi.x == dom_hi.x) { is_per[0] = 1;}
35  if (Geom(lev).isPeriodic(1) && sub_lo.y == dom_lo.y && sub_hi.y == dom_hi.y) { is_per[1] = 1;}
36 
37  int coord_sys = 0;
38 
39  // If subdomain == domain then we pass Geom(lev) to the FFT solver
40  if (subdomain == Geom(lev).Domain()) {
41  my_geom.define(Geom(lev).Domain(), Geom(lev).ProbDomain(), coord_sys, is_per);
42  } else {
43  // else we create a new geometry based only on the subdomain
44  // The information in my_geom used by the FFT routines is:
45  // 1) my_geom.Domain()
46  // 2) my_geom.CellSize()
47  // 3) my_geom.isAllPeriodic() / my_geom.periodicity()
48  RealBox rb( sub_lo.x *dx[0], sub_lo.y *dx[1], sub_lo.z *dx[2],
49  (sub_hi.x+1)*dx[0], (sub_hi.y+1)*dx[1], (sub_hi.z+1)*dx[2]);
50  my_geom.define(subdomain, rb, coord_sys, is_per);
51  }
52 
53  amrex::GMRES<MultiFab, TerrainPoisson> gmsolver;
54 
55  TerrainPoisson tp(my_geom, rhs.boxArray(), rhs.DistributionMap(), domain_bc_type,
56  stretched_dz_d[lev], ax_sub, ay_sub, az_sub, dJ_sub, &znd_sub,
58 
59  gmsolver.define(tp);
60 
61  gmsolver.setVerbose(mg_verbose);
62 
63  gmsolver.setRestartLength(50);
64 
65  tp.usePrecond(true);
66 
67  gmsolver.solve(phi, rhs, reltol, abstol);
68 
69  tp.getFluxes(phi, fluxes);
70 
71  for (MFIter mfi(phi); mfi.isValid(); ++mfi)
72  {
73  Box xbx = mfi.nodaltilebox(0);
74  Box ybx = mfi.nodaltilebox(1);
75  const Array4<Real >& fx_ar = fluxes[0].array(mfi);
76  const Array4<Real >& fy_ar = fluxes[1].array(mfi);
77  const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
78  const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
79  ParallelFor(xbx,ybx,
80  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
81  {
82  fx_ar(i,j,k) *= mf_ux(i,j,0);
83  },
84  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
85  {
86  fy_ar(i,j,k) *= mf_vy(i,j,0);
87  });
88  } // mfi
89 #else
90  amrex::ignore_unused(lev, rhs, phi, fluxes, ax_sub, ay_sub, az_sub, dJ_sub, znd_sub);
91 #endif
92 
93  // ****************************************************************************
94  // Impose bc's on pprime
95  // ****************************************************************************
96  ImposeBCsOnPhi(lev, phi, subdomain);
97 }

◆ solve_with_mlmg()

void ERF::solve_with_mlmg ( int  lev,
amrex::Vector< amrex::MultiFab > &  rhs,
amrex::Vector< amrex::MultiFab > &  p,
amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &  fluxes 
)

Solve the Poisson equation using MLMG Note that the level may or may not be level 0.

41 {
42  BL_PROFILE("ERF::solve_with_mlmg()");
43 
44  auto const dom_lo = lbound(geom[lev].Domain());
45  auto const dom_hi = ubound(geom[lev].Domain());
46 
47  LPInfo info;
48  // Allow a hidden direction if the domain is one cell wide in any lateral direction
49  if (dom_lo.x == dom_hi.x) {
50  info.setHiddenDirection(0);
51  } else if (dom_lo.y == dom_hi.y) {
52  info.setHiddenDirection(1);
53  }
54 
55  // Make sure the solver only sees the levels over which we are solving
56  Vector<BoxArray> ba_tmp; ba_tmp.push_back(rhs[0].boxArray());
57  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(rhs[0].DistributionMap());
58  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
59 
60  auto bclo = get_projection_bc(Orientation::low);
61  auto bchi = get_projection_bc(Orientation::high);
62 
63  // amrex::Print() << "BCLO " << bclo[0] << " " << bclo[1] << " " << bclo[2] << std::endl;
64  // amrex::Print() << "BCHI " << bchi[0] << " " << bchi[1] << " " << bchi[2] << std::endl;
65 
68 
69  // ****************************************************************************
70  // Multigrid solve
71  // ****************************************************************************
72 
73  MLPoisson mlpoisson(geom_tmp, ba_tmp, dm_tmp, info);
74  mlpoisson.setDomainBC(bclo, bchi);
75  if (lev > 0) {
76  mlpoisson.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
77  }
78  mlpoisson.setLevelBC(0, nullptr);
79 
80  // Use low order for outflow at physical boundaries
81  mlpoisson.setMaxOrder(2);
82 
83  MLMG mlmg(mlpoisson);
84  int max_iter = 100;
85  mlmg.setMaxIter(max_iter);
86 
87  mlmg.setVerbose(mg_verbose);
88  mlmg.setBottomVerbose(0);
89 
90  mlmg.solve(GetVecOfPtrs(phi),
91  GetVecOfConstPtrs(rhs),
92  reltol, abstol);
93  mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
94 
95  // ****************************************************************************
96  // Impose bc's on pprime
97  // ****************************************************************************
98  ImposeBCsOnPhi(lev, phi[0], geom[lev].Domain());
99 }

◆ sum_derived_quantities()

void ERF::sum_derived_quantities ( amrex::Real  time)
188 {
189  if (verbose <= 0 || NumDerDataLogs() <= 0) return;
190 
191  int lev = 0;
192 
193  AMREX_ALWAYS_ASSERT(lev == 0);
194 
195  // ************************************************************************
196  // WARNING: we are not filling ghost cells other than periodic outside the domain
197  // ************************************************************************
198 
199  MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
200  mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
201 
202  // Average all three components of velocity (on faces) to the cell center
203  average_face_to_cellcenter(mf_cc_vel,0,
204  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
205  &vars_new[lev][Vars::yvel],
206  &vars_new[lev][Vars::zvel]});
207  mf_cc_vel.FillBoundary(geom[lev].periodicity());
208 
209  if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
210  amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
211  }
212 
213  MultiFab r_wted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
214  MultiFab unwted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
215  MultiFab enstrophysq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
216  MultiFab theta_mf(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
217 
218 #ifdef _OPENMP
219 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
220 #endif
221  for (MFIter mfi(unwted_magvelsq, TilingIfNotGPU()); mfi.isValid(); ++mfi)
222  {
223  const Box& bx = mfi.tilebox();
224  auto& src_fab = mf_cc_vel[mfi];
225 
226  auto& dest1_fab = unwted_magvelsq[mfi];
227  derived::erf_dermagvelsq(bx, dest1_fab, 0, 1, src_fab, Geom(lev), t_new[0], nullptr, lev);
228 
229  auto& dest2_fab = enstrophysq[mfi];
230  derived::erf_derenstrophysq(bx, dest2_fab, 0, 1, src_fab, Geom(lev), t_new[0], nullptr, lev);
231  }
232 
233  // Copy the MF holding 1/2(u^2 + v^2 + w^2) into the MF that will hold 1/2 rho (u^2 + v^2 + w^2)d
234  MultiFab::Copy(r_wted_magvelsq, unwted_magvelsq, 0, 0, 1, 0);
235 
236  // Multiply the MF holding 1/2(u^2 + v^2 + w^2) by rho to get 1/2 rho (u^2 + v^2 + w^2)
237  MultiFab::Multiply(r_wted_magvelsq, vars_new[lev][Vars::cons], 0, 0, 1, 0);
238 
239  // Copy the MF holding (rho theta) into "theta_mf"
240  MultiFab::Copy(theta_mf, vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, 0);
241 
242  // Divide (rho theta) by rho to get theta in the MF "theta_mf"
243  MultiFab::Divide(theta_mf, vars_new[lev][Vars::cons], Rho_comp, 0, 1, 0);
244 
245  Real unwted_avg = volWgtSumMF(lev, unwted_magvelsq, 0, false);
246  Real r_wted_avg = volWgtSumMF(lev, r_wted_magvelsq, 0, false);
247  Real enstrsq_avg = volWgtSumMF(lev, enstrophysq, 0, false);
248  Real theta_avg = volWgtSumMF(lev, theta_mf, 0, false);
249 
250  // Get volume including terrain (consistent with volWgtSumMF routine)
251  MultiFab volume(grids[lev], dmap[lev], 1, 0);
252  auto const& dx = geom[lev].CellSizeArray();
253  Real cell_vol = dx[0]*dx[1]*dx[2];
254  volume.setVal(cell_vol);
255  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
256  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
257  }
258 #ifdef _OPENMP
259 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
260 #endif
261  for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
262  {
263  const Box& tbx = mfi.tilebox();
264  auto dst = volume.array(mfi);
265  const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
266  const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
267  ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
268  {
269  dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
270  });
271  }
272  Real vol = volume.sum();
273 
274  unwted_avg /= vol;
275  r_wted_avg /= vol;
276  enstrsq_avg /= vol;
277  theta_avg /= vol;
278 
279  const int nfoo = 4;
280  Real foo[nfoo] = {unwted_avg,r_wted_avg,enstrsq_avg,theta_avg};
281 #ifdef AMREX_LAZY
282  Lazy::QueueReduction([=]() mutable {
283 #endif
284  ParallelDescriptor::ReduceRealSum(
285  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
286 
287  if (ParallelDescriptor::IOProcessor()) {
288  int i = 0;
289  unwted_avg = foo[i++];
290  r_wted_avg = foo[i++];
291  enstrsq_avg = foo[i++];
292  theta_avg = foo[i++];
293 
294  std::ostream& data_log_der = DerDataLog(0);
295 
296  if (time == 0.0) {
297  data_log_der << std::setw(datwidth) << " time";
298  data_log_der << std::setw(datwidth) << " ke_den";
299  data_log_der << std::setw(datwidth) << " velsq";
300  data_log_der << std::setw(datwidth) << " enstrophy";
301  data_log_der << std::setw(datwidth) << " int_energy";
302  data_log_der << std::endl;
303  }
304  data_log_der << std::setw(datwidth) << std::setprecision(timeprecision) << time;
305  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << unwted_avg;
306  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << r_wted_avg;
307  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << enstrsq_avg;
308  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << theta_avg;
309  data_log_der << std::endl;
310 
311  } // if IOProcessor
312 #ifdef AMREX_LAZY
313  }
314 #endif
315 }
AMREX_FORCE_INLINE std::ostream & DerDataLog(int i)
Definition: ERF.H:1421
AMREX_FORCE_INLINE int NumDerDataLogs() noexcept
Definition: ERF.H:1435
void erf_dermagvelsq(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:346
void erf_derenstrophysq(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:284
Here is the call graph for this function:

◆ sum_energy_quantities()

void ERF::sum_energy_quantities ( amrex::Real  time)
319 {
320  if ( (verbose <= 0) || (tot_e_datalog.size() < 1) ) { return; }
321 
322  int lev = 0;
323 
324  AMREX_ALWAYS_ASSERT(lev == 0);
325 
326  // ************************************************************************
327  // WARNING: we are not filling ghost cells other than periodic outside the domain
328  // ************************************************************************
329 
330  MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
331  mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
332 
333  // Average all three components of velocity (on faces) to the cell center
334  average_face_to_cellcenter(mf_cc_vel,0,
335  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
336  &vars_new[lev][Vars::yvel],
337  &vars_new[lev][Vars::zvel]});
338  mf_cc_vel.FillBoundary(geom[lev].periodicity());
339 
340  if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
341  amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
342  }
343 
344  MultiFab tot_mass (grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
345  MultiFab tot_energy(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
346 
347  auto const& dx = geom[lev].CellSizeArray();
348  bool is_moist = (solverChoice.moisture_type != MoistureType::None);
349 
350 #ifdef _OPENMP
351 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
352 #endif
353  for (MFIter mfi(tot_mass, TilingIfNotGPU()); mfi.isValid(); ++mfi)
354  {
355  const Box& bx = mfi.tilebox();
356 
357  const Array4<Real>& cc_vel_arr = mf_cc_vel.array(mfi);
358  const Array4<Real>& tot_mass_arr = tot_mass.array(mfi);
359  const Array4<Real>& tot_energy_arr = tot_energy.array(mfi);
360  const Array4<const Real>& cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
361  const Array4<const Real>& z_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) :
362  Array4<const Real>{};
363  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
364  {
365  Real Qv = (is_moist) ? cons_arr(i,j,k,RhoQ1_comp) : 0.0;
366  Real Qc = (is_moist) ? cons_arr(i,j,k,RhoQ2_comp) : 0.0;
367  Real Qt = Qv + Qc;
368  Real Rhod = cons_arr(i,j,k,Rho_comp);
369  Real Rhot = Rhod * (1.0 + Qt);
370  Real Temp = getTgivenRandRTh(Rhod, cons_arr(i,j,k,RhoTheta_comp), Qv);
371  Real TKE = 0.5 * ( cc_vel_arr(i,j,k,0)*cc_vel_arr(i,j,k,0)
372  + cc_vel_arr(i,j,k,1)*cc_vel_arr(i,j,k,1)
373  + cc_vel_arr(i,j,k,2)*cc_vel_arr(i,j,k,2) );
374  Real zval = (z_arr) ? z_arr(i,j,k) : Real(k)*dx[2];
375 
376  Real Cv = Cp_d - R_d;
377  Real Cvv = Cp_v - R_v;
378  Real Cpv = Cp_v;
379 
380  tot_mass_arr(i,j,k) = Rhot;
381  tot_energy_arr(i,j,k) = Rhod * ( (Cv + Cvv*Qv + Cpv*Qc)*Temp - L_v*Qc
382  + (1.0 + Qt)*TKE + (1.0 + Qt)*CONST_GRAV*zval );
383 
384  });
385 
386  }
387 
388  Real tot_mass_avg = volWgtSumMF(lev, tot_mass , 0, false);
389  Real tot_energy_avg = volWgtSumMF(lev, tot_energy, 0, false);
390 
391  // Get volume including terrain (consistent with volWgtSumMF routine)
392  MultiFab volume(grids[lev], dmap[lev], 1, 0);
393  Real cell_vol = dx[0]*dx[1]*dx[2];
394  volume.setVal(cell_vol);
395  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
396  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
397  }
398 #ifdef _OPENMP
399 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
400 #endif
401  for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
402  {
403  const Box& tbx = mfi.tilebox();
404  auto dst = volume.array(mfi);
405  const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
406  const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
407  ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
408  {
409  dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
410  });
411  }
412  Real vol = volume.sum();
413 
414  // Divide by the volume
415  tot_mass_avg /= vol;
416  tot_energy_avg /= vol;
417 
418  const int nfoo = 2;
419  Real foo[nfoo] = {tot_mass_avg,tot_energy_avg};
420 #ifdef AMREX_LAZY
421  Lazy::QueueReduction([=]() mutable {
422 #endif
423  ParallelDescriptor::ReduceRealSum(
424  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
425 
426  if (ParallelDescriptor::IOProcessor()) {
427  int i = 0;
428  tot_mass_avg = foo[i++];
429  tot_energy_avg = foo[i++];
430 
431  std::ostream& data_log_energy = *tot_e_datalog[0];
432 
433  if (time == 0.0) {
434  data_log_energy << std::setw(datwidth) << " time";
435  data_log_energy << std::setw(datwidth) << " tot_mass";
436  data_log_energy << std::setw(datwidth) << " tot_energy";
437  data_log_energy << std::endl;
438  }
439  data_log_energy << std::setw(datwidth) << std::setprecision(timeprecision) << time;
440  data_log_energy << std::setw(datwidth) << std::setprecision(datprecision) << tot_mass_avg;
441  data_log_energy << std::setw(datwidth) << std::setprecision(datprecision) << tot_energy_avg;
442  data_log_energy << std::endl;
443 
444  } // if IOProcessor
445 #ifdef AMREX_LAZY
446  }
447 #endif
448 }
constexpr amrex::Real R_v
Definition: ERF_Constants.H:11
constexpr amrex::Real Cp_d
Definition: ERF_Constants.H:12
constexpr amrex::Real CONST_GRAV
Definition: ERF_Constants.H:21
constexpr amrex::Real Cp_v
Definition: ERF_Constants.H:13
constexpr amrex::Real R_d
Definition: ERF_Constants.H:10
constexpr amrex::Real L_v
Definition: ERF_Constants.H:16
Here is the call graph for this function:

◆ sum_integrated_quantities()

void ERF::sum_integrated_quantities ( amrex::Real  time)

Computes the integrated quantities on the grid such as the total scalar and total mass quantities. Prints and writes to output file.

Parameters
timeCurrent time
16 {
17  BL_PROFILE("ERF::sum_integrated_quantities()");
18 
19  if (verbose <= 0)
20  return;
21 
22  // Single level sum
23  Real mass_sl;
24 
25  // Multilevel sums
26  Real mass_ml = 0.0;
27  Real rhth_ml = 0.0;
28  Real scal_ml = 0.0;
29  Real mois_ml = 0.0;
30 
31 #if 1
32  mass_sl = volWgtSumMF(0,vars_new[0][Vars::cons],Rho_comp,false);
33  for (int lev = 0; lev <= finest_level; lev++) {
34  mass_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],Rho_comp,true);
35  }
36 #else
37  for (int lev = 0; lev <= finest_level; lev++) {
38  MultiFab pert_dens(vars_new[lev][Vars::cons].boxArray(),
39  vars_new[lev][Vars::cons].DistributionMap(),
40  1,0);
41  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
42  for ( MFIter mfi(pert_dens,TilingIfNotGPU()); mfi.isValid(); ++mfi)
43  {
44  const Box& bx = mfi.tilebox();
45  const Array4<Real >& pert_dens_arr = pert_dens.array(mfi);
46  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
47  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
48  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
49  pert_dens_arr(i, j, k, 0) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
50  });
51  }
52  if (lev == 0) {
53  mass_sl = volWgtSumMF(0,pert_dens,0,false);
54  }
55  mass_ml += volWgtSumMF(lev,pert_dens,0,true);
56  } // lev
57 #endif
58 
59  Real rhth_sl = volWgtSumMF(0,vars_new[0][Vars::cons], RhoTheta_comp,false);
60  Real scal_sl = volWgtSumMF(0,vars_new[0][Vars::cons],RhoScalar_comp,false);
61  Real mois_sl = 0.0;
62  if (solverChoice.moisture_type != MoistureType::None) {
63  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
64  for (int qoff(0); qoff<n_qstate_moist; ++qoff) {
65  mois_sl += volWgtSumMF(0,vars_new[0][Vars::cons],RhoQ1_comp+qoff,false);
66  }
67  }
68 
69  for (int lev = 0; lev <= finest_level; lev++) {
70  rhth_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons], RhoTheta_comp,true);
71  scal_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoScalar_comp,true);
72  if (solverChoice.moisture_type != MoistureType::None) {
73  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
74  for (int qoff(0); qoff<n_qstate_moist; ++qoff) {
75  mois_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoQ1_comp+qoff,false);
76  }
77  }
78  }
79 
80  Gpu::HostVector<Real> h_avg_ustar; h_avg_ustar.resize(1);
81  Gpu::HostVector<Real> h_avg_tstar; h_avg_tstar.resize(1);
82  Gpu::HostVector<Real> h_avg_olen; h_avg_olen.resize(1);
83  if ((m_SurfaceLayer != nullptr) && (NumDataLogs() > 0)) {
84  Box domain = geom[0].Domain();
85  int zdir = 2;
86  h_avg_ustar = sumToLine(*m_SurfaceLayer->get_u_star(0),0,1,domain,zdir);
87  h_avg_tstar = sumToLine(*m_SurfaceLayer->get_t_star(0),0,1,domain,zdir);
88  h_avg_olen = sumToLine(*m_SurfaceLayer->get_olen(0) ,0,1,domain,zdir);
89 
90  // Divide by the total number of cells we are averaging over
91  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
92  h_avg_ustar[0] /= area_z;
93  h_avg_tstar[0] /= area_z;
94  h_avg_olen[0] /= area_z;
95 
96  } else {
97  h_avg_ustar[0] = 0.;
98  h_avg_tstar[0] = 0.;
99  h_avg_olen[0] = 0.;
100  }
101 
102  const int nfoo = 8;
103  Real foo[nfoo] = {mass_sl,rhth_sl,scal_sl,mois_sl,mass_ml,rhth_ml,scal_ml,mois_ml};
104 #ifdef AMREX_LAZY
105  Lazy::QueueReduction([=]() mutable {
106 #endif
107  ParallelDescriptor::ReduceRealSum(
108  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
109 
110  if (ParallelDescriptor::IOProcessor()) {
111  int i = 0;
112  mass_sl = foo[i++];
113  rhth_sl = foo[i++];
114  scal_sl = foo[i++];
115  mois_sl = foo[i++];
116  mass_ml = foo[i++];
117  rhth_ml = foo[i++];
118  scal_ml = foo[i++];
119  mois_ml = foo[i++];
120 
121  Print() << '\n';
122  Print() << "TIME= " << std::setw(datwidth) << std::setprecision(timeprecision) << std::left << time << '\n';
123  if (finest_level == 0) {
124 #if 1
125  Print() << " MASS = " << mass_sl << '\n';
126 #else
127  Print() << " PERT MASS = " << mass_sl << '\n';
128 #endif
129  Print() << " RHO THETA = " << rhth_sl << '\n';
130  Print() << " RHO SCALAR = " << scal_sl << '\n';
131  Print() << " RHO QTOTAL = " << mois_sl << '\n';
132  } else {
133 #if 1
134  Print() << " MASS SL/ML = " << mass_sl << " " << mass_ml << '\n';
135 #else
136  Print() << " PERT MASS SL/ML = " << mass_sl << " " << mass_ml << '\n';
137 #endif
138  Print() << " RHO THETA SL/ML = " << rhth_sl << " " << rhth_ml << '\n';
139  Print() << " RHO SCALAR SL/ML = " << scal_sl << " " << scal_ml << '\n';
140  Print() << " RHO QTOTAL SL/ML = " << mois_sl << " " << mois_ml << '\n';
141  }
142 
143  // The first data log only holds scalars
144  if (NumDataLogs() > 0)
145  {
146  int n_d = 0;
147  std::ostream& data_log1 = DataLog(n_d);
148  if (data_log1.good()) {
149  if (time == 0.0) {
150  data_log1 << std::setw(datwidth) << " time";
151  data_log1 << std::setw(datwidth) << " u_star";
152  data_log1 << std::setw(datwidth) << " t_star";
153  data_log1 << std::setw(datwidth) << " olen";
154  data_log1 << std::endl;
155  } // time = 0
156 
157  // Write the quantities at this time
158  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time;
159  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_ustar[0];
160  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_tstar[0];
161  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_olen[0];
162  data_log1 << std::endl;
163  } // if good
164  } // loop over i
165  } // if IOProcessor
166 #ifdef AMREX_LAZY
167  });
168 #endif
169 
170  // This is just an alias for convenience
171  int lev = 0;
172  if (NumSamplePointLogs() > 0 && NumSamplePoints() > 0) {
173  for (int i = 0; i < NumSamplePoints(); ++i)
174  {
175  sample_points(lev, time, SamplePoint(i), vars_new[lev][Vars::cons]);
176  }
177  }
178  if (NumSampleLineLogs() > 0 && NumSampleLines() > 0) {
179  for (int i = 0; i < NumSampleLines(); ++i)
180  {
181  sample_lines(lev, time, SampleLine(i), vars_new[lev][Vars::cons]);
182  }
183  }
184 }
AMREX_FORCE_INLINE int NumSampleLineLogs() noexcept
Definition: ERF.H:1464
AMREX_FORCE_INLINE int NumSamplePointLogs() noexcept
Definition: ERF.H:1450
amrex::IntVect & SampleLine(int i)
Definition: ERF.H:1483
AMREX_FORCE_INLINE int NumSamplePoints() noexcept
Definition: ERF.H:1477
AMREX_FORCE_INLINE int NumSampleLines() noexcept
Definition: ERF.H:1490
amrex::IntVect & SamplePoint(int i)
Definition: ERF.H:1470
void sample_points(int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
Definition: ERF_WriteScalarProfiles.cpp:527
AMREX_FORCE_INLINE std::ostream & DataLog(int i)
Definition: ERF.H:1414
AMREX_FORCE_INLINE int NumDataLogs() noexcept
Definition: ERF.H:1428
void sample_lines(int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
Definition: ERF_WriteScalarProfiles.cpp:563

◆ timeStep()

void ERF::timeStep ( int  lev,
amrex::Real  time,
int  iteration 
)
private

Function that coordinates the evolution across levels – this calls Advance to do the actual advance at this level, then recursively calls itself at finer levels

Parameters
[in]levlevel of refinement (coarsest level is 0)
[in]timestart time for time advance
[in]iterationtime step counter
18 {
19  //
20  // We need to FillPatch the coarse level before assessing whether to regrid
21  // We have not done the swap yet so we fill the "new" which will become the "old"
22  //
23  MultiFab& S_new = vars_new[lev][Vars::cons];
24  MultiFab& U_new = vars_new[lev][Vars::xvel];
25  MultiFab& V_new = vars_new[lev][Vars::yvel];
26  MultiFab& W_new = vars_new[lev][Vars::zvel];
27 
28 #ifdef ERF_USE_NETCDF
29  //
30  // Since we now only read in a subset of the time slices in wrfbdy and
31  // wrflowinp, we need to check whether it's time to read in more.
32  //
33  bool use_moist = (solverChoice.moisture_type != MoistureType::None);
34  if (solverChoice.use_real_bcs && (lev==0)) {
35  Real dT = bdy_time_interval;
36 
37  int n_time_old = static_cast<int>( (time ) / dT);
38  int n_time_new = static_cast<int>( (time+dt[lev]) / dT);
39 
40  int ntimes = bdy_data_xlo.size();
41  for (int itime = 0; itime < ntimes; itime++)
42  {
43  //if (bdy_data_xlo[itime].size() > 0) {
44  // amrex::Print() << "HAVE DATA AT TIME " << itime << std::endl;
45  //} else {
46  // amrex::Print() << " NO DATA AT TIME " << itime << std::endl;
47  //}
48 
49  bool clear_itime = (itime < n_time_old);
50 
51  if (clear_itime && bdy_data_xlo[itime].size() > 0) {
52  bdy_data_xlo[itime].clear();
53  bdy_data_xhi[itime].clear();
54  bdy_data_ylo[itime].clear();
55  bdy_data_yhi[itime].clear();
56  //amrex::Print() << "CLEAR BDY DATA AT TIME " << itime << std::endl;
57  }
58 
59  bool need_itime = (itime >= n_time_old && itime <= n_time_new+1);
60  //if (need_itime) amrex::Print() << "NEED BDY DATA AT TIME " << itime << std::endl;
61 
62  if (bdy_data_xlo[itime].size() == 0 && need_itime) {
63  read_from_wrfbdy(itime,nc_bdy_file,geom[0].Domain(),
64  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
65  real_width);
66 
67  convert_all_wrfbdy_data(itime, geom[0].Domain(), bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi,
68  *mf_MUB, *mf_C1H, *mf_C2H,
70  geom[lev], use_moist);
71  }
72  } // itime
73  } // use_real_bcs && lev == 0
74 
75  if (!nc_low_file.empty() && (lev==0)) {
76  Real dT = low_time_interval;
77 
78  int n_time_old = static_cast<int>( (time ) / dT);
79  int n_time_new = static_cast<int>( (time+dt[lev]) / dT);
80 
81  int ntimes = bdy_data_xlo.size();
82  for (int itime = 0; itime < ntimes; itime++)
83  {
84  bool clear_itime = (itime < n_time_old);
85 
86  if (clear_itime && low_data_zlo[itime].size() > 0) {
87  low_data_zlo[itime].clear();
88  //amrex::Print() << "CLEAR LOW DATA AT TIME " << itime << std::endl;
89  }
90 
91  bool need_itime = (itime >= n_time_old && itime <= n_time_new+1);
92  //if (need_itime) amrex::Print() << "NEED LOW DATA AT TIME " << itime << std::endl;
93 
94  if (low_data_zlo[itime].size() == 0 && need_itime) {
95  read_from_wrflow(itime, nc_low_file, geom[lev].Domain(), low_data_zlo);
96 
97  update_sst_tsk(itime, geom[lev], ba2d[lev],
98  sst_lev[lev], tsk_lev[lev],
99  m_SurfaceLayer, low_data_zlo,
100  S_new, *mf_PSFC[lev],
101  solverChoice.rdOcp, lmask_lev[lev][0], use_moist);
102  }
103  } // itime
104  } // have nc_low_file && lev == 0
105 #endif
106 
107  //
108  // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
109  //
110  if (lev == 0) {
111  FillPatchCrseLevel(lev, time, {&S_new, &U_new, &V_new, &W_new});
112  } else if (lev < finest_level) {
113  FillPatchFineLevel(lev, time, {&S_new, &U_new, &V_new, &W_new},
114  {&S_new, &rU_new[lev], &rV_new[lev], &rW_new[lev]},
115  base_state[lev], base_state[lev]);
116  }
117 
118  if (regrid_int > 0) // We may need to regrid
119  {
120  // help keep track of whether a level was already regridded
121  // from a coarser level call to regrid
122  static Vector<int> last_regrid_step(max_level+1, 0);
123 
124  // regrid changes level "lev+1" so we don't regrid on max_level
125  // also make sure we don't regrid fine levels again if
126  // it was taken care of during a coarser regrid
127  if (lev < max_level)
128  {
129  if ( (istep[lev] % regrid_int == 0) && (istep[lev] > last_regrid_step[lev]) )
130  {
131  // regrid could add newly refine levels (if finest_level < max_level)
132  // so we save the previous finest level index
133  int old_finest = finest_level;
134 
135  regrid(lev, time);
136 
137 #ifdef ERF_USE_PARTICLES
138  if (finest_level != old_finest) {
139  particleData.Redistribute();
140  }
141 #endif
142 
143  // mark that we have regridded this level already
144  for (int k = lev; k <= finest_level; ++k) {
145  last_regrid_step[k] = istep[k];
146  }
147 
148  // if there are newly created levels, set the time step
149  for (int k = old_finest+1; k <= finest_level; ++k) {
150  dt[k] = dt[k-1] / static_cast<Real>(nsubsteps[k]);
151  }
152  } // if
153  } // lev
154  }
155 
156  // Update what we call "old" and "new" time
157  t_old[lev] = t_new[lev];
158  t_new[lev] += dt[lev];
159 
160  if (Verbose()) {
161  amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] ";
162  amrex::Print() << std::setprecision(timeprecision)
163  << "ADVANCE from elapsed time = " << t_old[lev] << " to " << t_new[lev]
164  << " with dt = " << dt[lev] << std::endl;
165  }
166 
167 #ifdef ERF_USE_WW3_COUPLING
168  amrex::Print() << " About to call send_to_ww3 from ERF_Timestep" << std::endl;
169  send_to_ww3(lev);
170  amrex::Print() << " About to call read_waves from ERF_Timestep" << std::endl;
171  read_waves(lev);
172  //send_to_ww3(lev);
173  //read_waves(lev);
174  //send_to_ww3(lev);
175 #endif
176 
177  // Advance a single level for a single time step
178  Advance(lev, time, dt[lev], istep[lev], nsubsteps[lev]);
179 
180  ++istep[lev];
181 
182  if (Verbose()) {
183  amrex::Print() << "[Level " << lev << " step " << istep[lev] << "] ";
184  amrex::Print() << "Advanced " << CountCells(lev) << " cells" << std::endl;
185  }
186 
187  if (lev < finest_level)
188  {
189  // recursive call for next-finer level
190  for (int i = 1; i <= nsubsteps[lev+1]; ++i)
191  {
192  Real strt_time_for_fine = time + (i-1)*dt[lev+1];
193  timeStep(lev+1, strt_time_for_fine, i);
194  }
195  }
196 
197  if (verbose && lev == 0 && solverChoice.moisture_type != MoistureType::None) {
198  amrex::Print() << "Cloud fraction " << time << " " << cloud_fraction(time) << std::endl;
199  }
200 }
amrex::Real cloud_fraction(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:451
void Advance(int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle)
Definition: ERF_Advance.cpp:20

◆ turbPert_amplitude()

void ERF::turbPert_amplitude ( const int  lev)
private
33 {
34  // Accessing data
35  auto& lev_new = vars_new[lev];
36 
37  // Creating local data
38  int ncons = lev_new[Vars::cons].nComp();
39  MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
40 
41  // Defining BoxArray type
42  auto m_ixtype = cons_data.boxArray().ixType();
43 
44 #ifdef _OPENMP
45 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
46 #endif
47  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi) {
48  const Box &bx = mfi.validbox();
49  const auto &cons_pert_arr = cons_data.array(mfi); // Address of perturbation array
50  const amrex::Array4<const amrex::Real> &pert_cell = turbPert.pb_cell[lev].array(mfi); // per-cell perturbation stored in structure
51 
52  turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cons_pert_arr, pert_cell);
53  } // mfi
54 }
Here is the call graph for this function:

◆ turbPert_update()

void ERF::turbPert_update ( const int  lev,
const amrex::Real  dt 
)
private
13 {
14  // Accessing data
15  auto& lev_new = vars_new[lev];
16 
17  // Create aliases to state data to pass to calc_tpi_update
18  int ncons = lev_new[Vars::cons].nComp();
19  MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
20  MultiFab xvel_data(lev_new[Vars::xvel], make_alias, 0, 1);
21  MultiFab yvel_data(lev_new[Vars::yvel], make_alias, 0, 1);
22 
23  // Computing perturbation update time
24  turbPert.calc_tpi_update(lev, local_dt, xvel_data, yvel_data, cons_data);
25 
26  Print() << "Successfully initialized turbulent perturbation update time and amplitude with type: "<< turbPert.pt_type <<"\n";
27 }
int pt_type
Definition: ERF_TurbPertStruct.H:631

◆ update_diffusive_arrays()

void ERF::update_diffusive_arrays ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
private
475 {
476  // ********************************************************************************************
477  // Diffusive terms
478  // ********************************************************************************************
479  bool l_use_terrain = (SolverChoice::terrain_type != TerrainType::None);
480  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
481  bool l_use_diff = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
482  l_use_kturb );
483  bool l_need_SmnSmn = solverChoice.turbChoice[lev].use_keqn;
484  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
485  bool l_rotate = ( solverChoice.use_rotate_surface_flux );
486 
487  bool l_implicit_diff = (solverChoice.vert_implicit_fac[0] > 0 ||
490 
491  BoxArray ba12 = convert(ba, IntVect(1,1,0));
492  BoxArray ba13 = convert(ba, IntVect(1,0,1));
493  BoxArray ba23 = convert(ba, IntVect(0,1,1));
494 
495  Tau[lev].resize(9);
496  Tau_corr[lev].resize(3);
497 
498  if (l_use_diff) {
499  //
500  // NOTE: We require ghost cells in the vertical when allowing grids that don't
501  // cover the entire vertical extent of the domain at this level
502  //
503  for (int i = 0; i < 3; i++) {
504  Tau[lev][i] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) );
505  }
506  Tau[lev][TauType::tau12] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
507  Tau[lev][TauType::tau13] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
508  Tau[lev][TauType::tau23] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
509  Tau[lev][TauType::tau12]->setVal(0.);
510  Tau[lev][TauType::tau13]->setVal(0.);
511  Tau[lev][TauType::tau23]->setVal(0.);
512  if (l_use_terrain) {
513  Tau[lev][TauType::tau21] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
514  Tau[lev][TauType::tau31] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
515  Tau[lev][TauType::tau32] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
516  Tau[lev][TauType::tau21]->setVal(0.);
517  Tau[lev][TauType::tau31]->setVal(0.);
518  Tau[lev][TauType::tau32]->setVal(0.);
519  } else if (l_implicit_diff) {
520  Tau[lev][TauType::tau31] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
521  Tau[lev][TauType::tau32] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
522  Tau[lev][TauType::tau31]->setVal(0.);
523  Tau[lev][TauType::tau32]->setVal(0.);
524  } else {
525  Tau[lev][TauType::tau21] = nullptr;
526  Tau[lev][TauType::tau31] = nullptr;
527  Tau[lev][TauType::tau32] = nullptr;
528  }
529 
530  if (l_implicit_diff && solverChoice.implicit_momentum_diffusion)
531  {
532  Tau_corr[lev][0] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) ); // Tau31
533  Tau_corr[lev][1] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) ); // Tau32
534  Tau_corr[lev][0]->setVal(0.);
535  Tau_corr[lev][1]->setVal(0.);
536 #ifdef ERF_IMPLICIT_W
537  Tau_corr[lev][2] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) ); // Tau33
538  Tau_corr[lev][2]->setVal(0.);
539 #else
540  Tau_corr[lev][2] = nullptr;
541 #endif
542  } else {
543  Tau_corr[lev][0] = nullptr;
544  Tau_corr[lev][1] = nullptr;
545  Tau_corr[lev][2] = nullptr;
546  }
547 
548  SFS_hfx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
549  SFS_hfx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
550  SFS_hfx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
551  SFS_diss_lev[lev] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) );
552  SFS_hfx1_lev[lev]->setVal(0.);
553  SFS_hfx2_lev[lev]->setVal(0.);
554  SFS_hfx3_lev[lev]->setVal(0.);
555  SFS_diss_lev[lev]->setVal(0.);
556  if (l_use_moist) {
557  SFS_q1fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
558  SFS_q2fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
559  SFS_q1fx3_lev[lev]->setVal(0.0);
560  SFS_q2fx3_lev[lev]->setVal(0.0);
561  if (l_rotate) {
562  SFS_q1fx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
563  SFS_q1fx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
564  SFS_q1fx1_lev[lev]->setVal(0.0);
565  SFS_q1fx2_lev[lev]->setVal(0.0);
566  } else {
567  SFS_q1fx1_lev[lev] = nullptr;
568  SFS_q1fx2_lev[lev] = nullptr;
569  }
570  } else {
571  SFS_q1fx1_lev[lev] = nullptr;
572  SFS_q1fx2_lev[lev] = nullptr;
573  SFS_q1fx3_lev[lev] = nullptr;
574  SFS_q2fx3_lev[lev] = nullptr;
575  }
576  } else {
577  for (int i = 0; i < 9; i++) {
578  Tau[lev][i] = nullptr;
579  }
580  SFS_hfx1_lev[lev] = nullptr; SFS_hfx2_lev[lev] = nullptr; SFS_hfx3_lev[lev] = nullptr;
581  SFS_diss_lev[lev] = nullptr;
582  }
583 
584  if (l_use_kturb) {
585  eddyDiffs_lev[lev] = std::make_unique<MultiFab>(ba, dm, EddyDiff::NumDiffs, 2);
586  eddyDiffs_lev[lev]->setVal(0.0);
587  if(l_need_SmnSmn) {
588  SmnSmn_lev[lev] = std::make_unique<MultiFab>( ba, dm, 1, 0 );
589  } else {
590  SmnSmn_lev[lev] = nullptr;
591  }
592  } else {
593  eddyDiffs_lev[lev] = nullptr;
594  SmnSmn_lev[lev] = nullptr;
595  }
596 }
@ NumDiffs
Definition: ERF_IndexDefines.H:181

◆ update_terrain_arrays()

void ERF::update_terrain_arrays ( int  lev)
714 {
715  if (SolverChoice::mesh_type == MeshType::StretchedDz ||
716  SolverChoice::mesh_type == MeshType::VariableDz) {
717  make_J(geom[lev],*z_phys_nd[lev],*detJ_cc[lev]);
718  make_areas(geom[lev],*z_phys_nd[lev],*ax[lev],*ay[lev],*az[lev]);
719  make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
720  } else { // MeshType::ConstantDz
721  if (SolverChoice::terrain_type == TerrainType::EB) {
722  const auto& ebfact = *eb[lev]->get_const_factory();
723  const MultiFab& volfrac = ebfact.getVolFrac();
724  detJ_cc[lev] = std::make_unique<MultiFab>(volfrac, amrex::make_alias, 0, volfrac.nComp());
725  }
726  }
727 }
void make_areas(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &ax, MultiFab &ay, MultiFab &az)
Definition: ERF_TerrainMetrics.cpp:559
void make_J(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &detJ_cc)
Definition: ERF_TerrainMetrics.cpp:521
Here is the call graph for this function:

◆ volWgtSumMF()

Real ERF::volWgtSumMF ( int  lev,
const amrex::MultiFab &  mf,
int  comp,
bool  finemask 
)

Utility function for computing a volume weighted sum of MultiFab data for a single component

Parameters
levCurrent level
mfMultiFab on which we do the volume weighted sum
compIndex of the component we want to sum
localBoolean sets whether or not to reduce the sum over the domain (false) or compute sums local to each MPI rank (true)
finemaskIf a finer level is available, determines whether we mask fine data
21 {
22  BL_PROFILE("ERF::volWgtSumMF()");
23 
24  Real sum = 0.0;
25  MultiFab tmp(grids[lev], dmap[lev], 1, 0);
26  MultiFab::Copy(tmp, mf, comp, 0, 1, 0);
27 
28  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
29  // m is the map scale factor at cell centers
30 #ifdef _OPENMP
31 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
32 #endif
33  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
34  const Box& bx = mfi.tilebox();
35  const auto dst = tmp.array(mfi);
36  const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
37  const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
38  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
39  {
40  dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
41  });
42  } // mfi
43 
44  if (lev < finest_level && finemask) {
45  const MultiFab& mask = build_fine_mask(lev+1);
46  MultiFab::Multiply(tmp, mask, 0, 0, 1, 0);
47  }
48 
49  // Get volume including terrain (consistent with volWgtSumMF routine)
50  MultiFab volume(grids[lev], dmap[lev], 1, 0);
51  auto const& dx = geom[lev].CellSizeArray();
52  Real cell_vol = dx[0]*dx[1]*dx[2];
53  volume.setVal(cell_vol);
54  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
55  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
56  }
57 
58  //
59  // Note that when we send in local = true, NO ParallelAllReduce::Sum
60  // is called inside the Dot product -- we will do that before we print
61  //
62  bool local = true;
63  sum = MultiFab::Dot(tmp, 0, volume, 0, 1, 0, local);
64 
65  return sum;
66 }
amrex::MultiFab & build_fine_mask(int lev)
Definition: ERF_VolWgtSum.cpp:76

◆ WeatherDataInterpolation()

void ERF::WeatherDataInterpolation ( const amrex::Real  time)
507 {
508 
509  static Real next_read_forecast_time = -1.0;
510  Real hindcast_data_interval = solverChoice.hindcast_data_interval_in_hrs*3600.0;
511 
512  if (next_read_forecast_time < 0.0) {
513  int next_multiple = static_cast<int>(time / hindcast_data_interval);
514  next_read_forecast_time = next_multiple * hindcast_data_interval;
515  }
516  if (time >= next_read_forecast_time) {
517 
518  std::string folder = solverChoice.hindcast_boundary_data_dir;
519 
520  // Check if folder exists and is a directory
521  if (!fs::exists(folder) || !fs::is_directory(folder)) {
522  throw std::runtime_error("Error: Folder '" + folder + "' does not exist or is not a directory.");
523  }
524 
525  std::vector<std::string> bin_files;
526 
527  for (const auto& entry : fs::directory_iterator(folder)) {
528  if (!entry.is_regular_file()) continue;
529 
530  std::string fname = entry.path().filename().string();
531  if (fname.size() >= 4 && fname.substr(fname.size() - 4) == ".bin") {
532  bin_files.push_back(entry.path().string());
533  }
534  }
535  std::sort(bin_files.begin(), bin_files.end());
536 
537  // Check if no .bin files were found
538  if (bin_files.empty()) {
539  throw std::runtime_error("Error: No .bin files found in folder '" + folder + "'.");
540  }
541 
542  std::string filename1, filename2;
543 
544  int idx1 = static_cast<int>(time / hindcast_data_interval);
545  int idx2 = static_cast<int>(time / hindcast_data_interval)+1;
546  std::cout << "Reading weather data " << time << " " << idx1 << " " << idx2 <<" " << bin_files.size() << std::endl;
547 
548  if (idx2 >= static_cast<int>(bin_files.size())) {
549  throw std::runtime_error("Error: Not enough .bin files to cover time " + std::to_string(time));
550  }
551 
552  filename1 = bin_files[idx1];
553  filename2 = bin_files[idx2];
554 
555  BoxArray nba;
556  DistributionMapping dm;
557  Geometry geom_weather;
558 
559  //Read in weather_forecast_1
561  geom_weather,
562  nba,
563  dm);
564 
565  FillWeatherDataMultiFab(filename1,
566  geom_weather,
567  nba,
568  dm,
570 
573 
574  FillWeatherDataMultiFab(filename2,
575  geom_weather,
576  nba,
577  dm,
581 
583 
584  next_read_forecast_time += hindcast_data_interval;
585  }
586  Real alpha1 = 1.0 - (time - next_read_forecast_time)/hindcast_data_interval;
587  Real alpha2 = 1.0 - alpha1;
588 
589  MultiFab& erf_mf_cons = forecast_state_interp[0][Vars::cons];
590  MultiFab& erf_mf_xvel = forecast_state_interp[0][Vars::xvel];
591  MultiFab& erf_mf_yvel = forecast_state_interp[0][Vars::yvel];
592  //MultiFab& erf_mf_zvel = forecast_state_interp[0][Vars::zvel];
593  MultiFab& erf_mf_latlon = forecast_state_interp[0][4];
594 
595  MultiFab::LinComb(forecast_state_interp[0][Vars::cons],
596  alpha1, forecast_state_1[0][Vars::cons], 0,
597  alpha2, forecast_state_2[0][Vars::cons], 0,
598  0, erf_mf_cons.nComp(), forecast_state_interp[0][Vars::cons].nGrow());
599  MultiFab::LinComb(forecast_state_interp[0][Vars::xvel],
600  alpha1, forecast_state_1[0][Vars::xvel], 0,
601  alpha2, forecast_state_2[0][Vars::xvel], 0,
602  0, erf_mf_xvel.nComp(), forecast_state_interp[0][Vars::xvel].nGrow());
603  MultiFab::LinComb(forecast_state_interp[0][Vars::yvel],
604  alpha1, forecast_state_1[0][Vars::yvel], 0,
605  alpha2, forecast_state_2[0][Vars::yvel], 0,
606  0, erf_mf_yvel.nComp(), forecast_state_interp[0][Vars::yvel].nGrow());
607  MultiFab::LinComb(forecast_state_interp[0][4],
608  alpha1, forecast_state_1[0][4], 0,
609  alpha2, forecast_state_2[0][4], 0,
610  0, erf_mf_latlon.nComp(), forecast_state_interp[0][4].nGrow());
611 
612 
613  /*Vector<std::string> varnames_plot_mf = {
614  "rho", "rhotheta", "rhoqv", "rhoqc", "rhoqr", "xvel", "yvel", "zvel", "latitude", "longitude"
615  }; // Customize variable names
616 
617  std::string pltname = "plt_interp";
618 
619  MultiFab plot_mf(erf_mf_cons.boxArray(), erf_mf_cons.DistributionMap(),
620  10, 0);
621 
622  plot_mf.setVal(0.0);
623 
624  for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
625  const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
626  const Array4<Real> &erf_mf_cons_arr = erf_mf_cons.array(mfi);
627  const Array4<Real> &erf_mf_xvel_arr = erf_mf_xvel.array(mfi);
628  const Array4<Real> &erf_mf_yvel_arr = erf_mf_yvel.array(mfi);
629  const Array4<Real> &erf_mf_zvel_arr = erf_mf_zvel.array(mfi);
630  const Array4<Real> &erf_mf_latlon_arr = erf_mf_latlon.array(mfi);
631 
632  const Box& bx = mfi.validbox();
633 
634  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
635  plot_mf_arr(i,j,k,0) = erf_mf_cons_arr(i,j,k,Rho_comp);
636  plot_mf_arr(i,j,k,1) = erf_mf_cons_arr(i,j,k,RhoTheta_comp);
637  plot_mf_arr(i,j,k,2) = erf_mf_cons_arr(i,j,k,RhoQ1_comp);
638  plot_mf_arr(i,j,k,3) = erf_mf_cons_arr(i,j,k,RhoQ2_comp);
639  plot_mf_arr(i,j,k,4) = erf_mf_cons_arr(i,j,k,RhoQ3_comp);
640 
641  plot_mf_arr(i,j,k,5) = (erf_mf_xvel_arr(i,j,k,0) + erf_mf_xvel_arr(i+1,j,k,0))/2.0;
642  plot_mf_arr(i,j,k,6) = (erf_mf_yvel_arr(i,j,k,0) + erf_mf_yvel_arr(i,j+1,k,0))/2.0;
643  plot_mf_arr(i,j,k,7) = (erf_mf_zvel_arr(i,j,k,0) + erf_mf_zvel_arr(i,j,k+1,0))/2.0;
644 
645  plot_mf_arr(i,j,k,8) = erf_mf_latlon_arr(i,j,k,0);
646  plot_mf_arr(i,j,k,9) = erf_mf_latlon_arr(i,j,k,1);
647  });
648  }
649 
650 
651  WriteSingleLevelPlotfile(
652  pltname,
653  plot_mf,
654  varnames_plot_mf,
655  geom[0],
656  time,
657  0 // level
658  );*/
659 
660 
661 
662 }
void CreateWeatherDataGeomBoxArrayDistMap(const std::string &filename, amrex::Geometry &geom_weather, amrex::BoxArray &nba, amrex::DistributionMapping &dm)
Definition: ERF_WeatherDataInterpolation.cpp:155
void CreateForecastStateMultiFabs(amrex::Vector< amrex::Vector< amrex::MultiFab >> &forecast_state)
Definition: ERF_WeatherDataInterpolation.cpp:249
void FillWeatherDataMultiFab(const std::string &filename, const amrex::Geometry &geom_weather, const amrex::BoxArray &nba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &weather_forecast_data)
Definition: ERF_WeatherDataInterpolation.cpp:438
amrex::Vector< amrex::MultiFab > weather_forecast_data_2
Definition: ERF.H:158
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_2
Definition: ERF.H:160
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_1
Definition: ERF.H:159
amrex::Vector< amrex::MultiFab > weather_forecast_data_1
Definition: ERF.H:158
void InterpWeatherDataOntoMesh(const amrex::Geometry &geom_weather, amrex::MultiFab &weather_forecast_interp, amrex::Vector< amrex::Vector< amrex::MultiFab >> &forecast_state)
Definition: ERF_WeatherDataInterpolation.cpp:268
amrex::Real hindcast_data_interval_in_hrs
Definition: ERF_DataStruct.H:1070
std::string hindcast_boundary_data_dir
Definition: ERF_DataStruct.H:1069

◆ Write2DPlotFile()

void ERF::Write2DPlotFile ( int  which,
PlotFileType  plotfile_type,
amrex::Vector< std::string >  plot_var_names 
)
1920 {
1921  const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
1922  const int ncomp_mf = varnames.size();
1923 
1924  if (ncomp_mf == 0) return;
1925 
1926  // Vector of MultiFabs for cell-centered data
1927  Vector<MultiFab> mf(finest_level+1);
1928  for (int lev = 0; lev <= finest_level; ++lev) {
1929  mf[lev].define(ba2d[lev], dmap[lev], ncomp_mf, 0);
1930  }
1931 
1932 
1933  // **********************************************************************************************
1934  // (Effectively) 2D arrays
1935  // **********************************************************************************************
1936  for (int lev = 0; lev <= finest_level; ++lev)
1937  {
1938  // Make sure getPgivenRTh and getTgivenRandRTh don't fail
1939  if (check_for_nans) {
1941  }
1942 
1943  int mf_comp = 0;
1944 
1945  // Set all components to zero in case they aren't defined below
1946  mf[lev].setVal(0.0);
1947 
1948  // Expose domain khi and klo at each level
1949  int klo = geom[lev].Domain().smallEnd(2);
1950  int khi = geom[lev].Domain().bigEnd(2);
1951 
1952  if (containerHasElement(plot_var_names, "z_surf")) {
1953 #ifdef _OPENMP
1954 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1955 #endif
1956  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1957  {
1958  const Box& bx = mfi.tilebox();
1959  const Array4<Real>& derdat = mf[lev].array(mfi);
1960  const Array4<const Real>& z_phys_arr = z_phys_nd[lev]->const_array(mfi);
1961  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
1962  derdat(i, j, k, mf_comp) = Compute_Z_AtWFace(i, j, 0, z_phys_arr);
1963  });
1964  }
1965  mf_comp++;
1966  }
1967 
1968  if (containerHasElement(plot_var_names, "landmask")) {
1969 #ifdef _OPENMP
1970 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1971 #endif
1972  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1973  {
1974  const Box& bx = mfi.tilebox();
1975  const Array4<Real>& derdat = mf[lev].array(mfi);
1976  const Array4<const int>& lmask_arr = lmask_lev[lev][0]->const_array(mfi);
1977  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
1978  derdat(i, j, k, mf_comp) = lmask_arr(i, j, 0);
1979  });
1980  }
1981  mf_comp++;
1982  }
1983 
1984  if (containerHasElement(plot_var_names, "mapfac")) {
1985 #ifdef _OPENMP
1986 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1987 #endif
1988  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1989  {
1990  const Box& bx = mfi.tilebox();
1991  const Array4<Real>& derdat = mf[lev].array(mfi);
1992  const Array4<Real>& mf_m = mapfac[lev][MapFacType::m_x]->array(mfi);
1993  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
1994  derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
1995  });
1996  }
1997  mf_comp++;
1998  }
1999 
2000  if (containerHasElement(plot_var_names, "lat_m")) {
2001  if (lat_m[lev]) {
2002 #ifdef _OPENMP
2003 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2004 #endif
2005  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2006  {
2007  const Box& bx = mfi.tilebox();
2008  const Array4<Real>& derdat = mf[lev].array(mfi);
2009  const Array4<Real>& data = lat_m[lev]->array(mfi);
2010  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2011  derdat(i, j, k, mf_comp) = data(i,j,0);
2012  });
2013  }
2014  }
2015  mf_comp++;
2016  } // lat_m
2017 
2018  if (containerHasElement(plot_var_names, "lon_m")) {
2019  if (lon_m[lev]) {
2020 #ifdef _OPENMP
2021 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2022 #endif
2023  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2024  {
2025  const Box& bx = mfi.tilebox();
2026  const Array4<Real>& derdat = mf[lev].array(mfi);
2027  const Array4<Real>& data = lon_m[lev]->array(mfi);
2028  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2029  derdat(i, j, k, mf_comp) = data(i,j,0);
2030  });
2031  }
2032  } else {
2033  mf[lev].setVal(0.0,mf_comp,1,0);
2034  }
2035 
2036  mf_comp++;
2037 
2038  } // lon_m
2039 
2040  ///////////////////////////////////////////////////////////////////////
2041  // These quantities are diagnosed by the surface layer
2042  if (containerHasElement(plot_var_names, "u_star")) {
2043 #ifdef _OPENMP
2044 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2045 #endif
2046  if (m_SurfaceLayer) {
2047  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2048  {
2049  const Box& bx = mfi.tilebox();
2050  const auto& derdat = mf[lev].array(mfi);
2051  const auto& ustar = m_SurfaceLayer->get_u_star(lev)->const_array(mfi);
2052  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2053  derdat(i, j, k, mf_comp) = ustar(i, j, 0);
2054  });
2055  }
2056  } else {
2057  mf[lev].setVal(-999,mf_comp,1,0);
2058  }
2059  mf_comp++;
2060  } // ustar
2061 
2062  if (containerHasElement(plot_var_names, "w_star")) {
2063 #ifdef _OPENMP
2064 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2065 #endif
2066  if (m_SurfaceLayer) {
2067  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2068  {
2069  const Box& bx = mfi.tilebox();
2070  const auto& derdat = mf[lev].array(mfi);
2071  const auto& ustar = m_SurfaceLayer->get_w_star(lev)->const_array(mfi);
2072  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2073  derdat(i, j, k, mf_comp) = ustar(i, j, 0);
2074  });
2075  }
2076  } else {
2077  mf[lev].setVal(-999,mf_comp,1,0);
2078  }
2079  mf_comp++;
2080  } // wstar
2081 
2082  if (containerHasElement(plot_var_names, "t_star")) {
2083 #ifdef _OPENMP
2084 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2085 #endif
2086  if (m_SurfaceLayer) {
2087  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2088  {
2089  const Box& bx = mfi.tilebox();
2090  const auto& derdat = mf[lev].array(mfi);
2091  const auto& ustar = m_SurfaceLayer->get_t_star(lev)->const_array(mfi);
2092  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2093  derdat(i, j, k, mf_comp) = ustar(i, j, 0);
2094  });
2095  }
2096  } else {
2097  mf[lev].setVal(-999,mf_comp,1,0);
2098  }
2099  mf_comp++;
2100  } // tstar
2101 
2102  if (containerHasElement(plot_var_names, "q_star")) {
2103 #ifdef _OPENMP
2104 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2105 #endif
2106  if (m_SurfaceLayer) {
2107  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2108  {
2109  const Box& bx = mfi.tilebox();
2110  const auto& derdat = mf[lev].array(mfi);
2111  const auto& ustar = m_SurfaceLayer->get_q_star(lev)->const_array(mfi);
2112  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2113  derdat(i, j, k, mf_comp) = ustar(i, j, 0);
2114  });
2115  }
2116  } else {
2117  mf[lev].setVal(-999,mf_comp,1,0);
2118  }
2119  mf_comp++;
2120  } // qstar
2121 
2122  if (containerHasElement(plot_var_names, "Olen")) {
2123 #ifdef _OPENMP
2124 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2125 #endif
2126  if (m_SurfaceLayer) {
2127  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2128  {
2129  const Box& bx = mfi.tilebox();
2130  const auto& derdat = mf[lev].array(mfi);
2131  const auto& ustar = m_SurfaceLayer->get_olen(lev)->const_array(mfi);
2132  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2133  derdat(i, j, k, mf_comp) = ustar(i, j, 0);
2134  });
2135  }
2136  } else {
2137  mf[lev].setVal(-999,mf_comp,1,0);
2138  }
2139  mf_comp++;
2140  } // Olen
2141 
2142  if (containerHasElement(plot_var_names, "pblh")) {
2143 #ifdef _OPENMP
2144 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2145 #endif
2146  if (m_SurfaceLayer) {
2147  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2148  {
2149  const Box& bx = mfi.tilebox();
2150  const auto& derdat = mf[lev].array(mfi);
2151  const auto& ustar = m_SurfaceLayer->get_pblh(lev)->const_array(mfi);
2152  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2153  derdat(i, j, k, mf_comp) = ustar(i, j, 0);
2154  });
2155  }
2156  } else {
2157  mf[lev].setVal(-999,mf_comp,1,0);
2158  }
2159  mf_comp++;
2160  } // pblh
2161 
2162  if (containerHasElement(plot_var_names, "t_surf")) {
2163 #ifdef _OPENMP
2164 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2165 #endif
2166  if (m_SurfaceLayer) {
2167  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2168  {
2169  const Box& bx = mfi.tilebox();
2170  const auto& derdat = mf[lev].array(mfi);
2171  const auto& ustar = m_SurfaceLayer->get_t_surf(lev)->const_array(mfi);
2172  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2173  derdat(i, j, k, mf_comp) = ustar(i, j, 0);
2174  });
2175  }
2176  } else {
2177  mf[lev].setVal(-999,mf_comp,1,0);
2178  }
2179  mf_comp++;
2180  } // tsurf
2181 
2182  if (containerHasElement(plot_var_names, "q_surf")) {
2183 #ifdef _OPENMP
2184 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2185 #endif
2186  if (m_SurfaceLayer) {
2187  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2188  {
2189  const Box& bx = mfi.tilebox();
2190  const auto& derdat = mf[lev].array(mfi);
2191  const auto& ustar = m_SurfaceLayer->get_q_surf(lev)->const_array(mfi);
2192  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2193  derdat(i, j, k, mf_comp) = ustar(i, j, 0);
2194  });
2195  }
2196  } else {
2197  mf[lev].setVal(-999,mf_comp,1,0);
2198  }
2199  mf_comp++;
2200  } // qsurf
2201 
2202  if (containerHasElement(plot_var_names, "z0")) {
2203 #ifdef _OPENMP
2204 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2205 #endif
2206  if (m_SurfaceLayer) {
2207  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2208  {
2209  const Box& bx = mfi.tilebox();
2210  const auto& derdat = mf[lev].array(mfi);
2211  const auto& ustar = m_SurfaceLayer->get_z0(lev)->const_array(mfi);
2212  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2213  derdat(i, j, k, mf_comp) = ustar(i, j, 0);
2214  });
2215  }
2216  } else {
2217  mf[lev].setVal(-999,mf_comp,1,0);
2218  }
2219  mf_comp++;
2220  } // z0
2221 
2222  if (containerHasElement(plot_var_names, "OLR")) {
2223 #ifdef _OPENMP
2224 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2225 #endif
2226  if (solverChoice.rad_type != RadiationType::None) {
2227  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2228  {
2229  const Box& bx = mfi.tilebox();
2230  const auto& derdat = mf[lev].array(mfi);
2231  const auto& olr = rad_fluxes[lev]->const_array(mfi);
2232  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2233  derdat(i, j, k, mf_comp) = olr(i, j, khi, 2);
2234  });
2235  }
2236  } else {
2237  mf[lev].setVal(-999,mf_comp,1,0);
2238  }
2239  mf_comp++;
2240  } // OLR
2241 
2242  if (containerHasElement(plot_var_names, "sens_flux")) {
2243 #ifdef _OPENMP
2244 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2245 #endif
2246  if (SFS_hfx3_lev[lev]) {
2247  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2248  {
2249  const Box& bx = mfi.tilebox();
2250  const auto& derdat = mf[lev].array(mfi);
2251  const auto& hfx_arr = SFS_hfx3_lev[lev]->const_array(mfi);
2252  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2253  derdat(i, j, k, mf_comp) = hfx_arr(i, j, klo);
2254  });
2255  }
2256  } else {
2257  mf[lev].setVal(-999,mf_comp,1,0);
2258  }
2259  mf_comp++;
2260  } // sens_flux
2261 
2262  if (containerHasElement(plot_var_names, "laten_flux")) {
2263 #ifdef _OPENMP
2264 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2265 #endif
2266  if (SFS_hfx3_lev[lev]) {
2267  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2268  {
2269  const Box& bx = mfi.tilebox();
2270  const auto& derdat = mf[lev].array(mfi);
2271  const auto& qfx_arr = SFS_q1fx3_lev[lev]->const_array(mfi);
2272  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2273  derdat(i, j, k, mf_comp) = qfx_arr(i, j, klo);
2274  });
2275  }
2276  } else {
2277  mf[lev].setVal(-999,mf_comp,1,0);
2278  }
2279  mf_comp++;
2280  } // laten_flux
2281 
2282  if (containerHasElement(plot_var_names, "surf_pres")) {
2283  bool moist = (solverChoice.moisture_type != MoistureType::None);
2284 #ifdef _OPENMP
2285 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
2286 #endif
2287  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
2288  {
2289  const Box& bx = mfi.tilebox();
2290  const auto& derdat = mf[lev].array(mfi);
2291  const auto& cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
2292  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
2293  auto rt = cons_arr(i,j,klo,RhoTheta_comp);
2294  auto qv = (moist) ? cons_arr(i,j,klo,RhoQ1_comp)/cons_arr(i,j,klo,Rho_comp)
2295  : 0.0;
2296  derdat(i, j, k, mf_comp) = getPgivenRTh(rt, qv);
2297  });
2298  }
2299  mf_comp++;
2300  } // surf_pres
2301 
2302  } // lev
2303 
2304  std::string plotfilename;
2305  if (which == 1) {
2306  plotfilename = Concatenate(plot2d_file_1, istep[0], 5);
2307  } else if (which == 2) {
2308  plotfilename = Concatenate(plot2d_file_2, istep[0], 5);
2309  }
2310 
2311  Vector<Geometry> my_geom(finest_level+1);
2312 
2313  Array<int,AMREX_SPACEDIM> is_per; is_per[0] = 0; is_per[1] = 0; is_per[2] = 0;
2314  if (geom[0].isPeriodic(0)) { is_per[0] = 1;}
2315  if (geom[0].isPeriodic(1)) { is_per[1] = 1;}
2316 
2317  int coord_sys = 0;
2318 
2319  for (int lev = 0; lev <= finest_level; lev++)
2320  {
2321  Box slab = makeSlab(geom[lev].Domain(),2,0);
2322  auto const slab_lo = lbound(slab);
2323  auto const slab_hi = ubound(slab);
2324 
2325  // Create a new geometry based only on the 2D slab
2326  // We need
2327  // 1) my_geom.Domain()
2328  // 2) my_geom.CellSize()
2329  // 3) my_geom.periodicity()
2330  const auto dx = geom[lev].CellSize();
2331  RealBox rb( slab_lo.x *dx[0], slab_lo.y *dx[1], slab_lo.z *dx[2],
2332  (slab_hi.x+1)*dx[0], (slab_hi.y+1)*dx[1], (slab_hi.z+1)*dx[2]);
2333  my_geom[lev].define(slab, rb, coord_sys, is_per);
2334  }
2335 
2336  if (plotfile_type == PlotFileType::Amrex)
2337  {
2338  Print() << "Writing 2D native plotfile " << plotfilename << "\n";
2339  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
2340  GetVecOfConstPtrs(mf),
2341  varnames, my_geom, t_new[0], istep, refRatio());
2342  writeJobInfo(plotfilename);
2343 
2344 #ifdef ERF_USE_NETCDF
2345  } else if (plotfile_type == PlotFileType::Netcdf) {
2346  int lev = 0;
2347  int l_which = 0;
2348  const Real* p_lo = my_geom[lev].ProbLo();
2349  const Real* p_hi = my_geom[lev].ProbHi();
2350  const auto dx = my_geom[lev].CellSize();
2351  writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
2352  {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],dx[2]}, {dx[0],dx[1],dx[2]},
2353  my_geom[lev].Domain(), t_new[0], start_bdy_time);
2354 #endif
2355  } else {
2356  // Here we assume the plotfile_type is PlotFileType::None
2357  Print() << "Writing no 2D plotfile since plotfile_type is none" << std::endl;
2358  }
2359 }
void writeNCPlotFile(int lev, int which_subdomain, const std::string &dir, const Vector< const MultiFab * > &plotMF, const Vector< std::string > &plot_var_names, const Vector< int > &, Array< Real, AMREX_SPACEDIM > prob_lo, Array< Real, AMREX_SPACEDIM > prob_hi, Array< Real, AMREX_SPACEDIM > dx_in, const Box &subdomain, const Real time, const Real start_bdy_time)
Definition: ERF_NCPlotFile.cpp:14
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_Z_AtWFace(const int &i, const int &j, const int &k, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:374
static amrex::Vector< std::string > PlotFileVarNames(amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile.cpp:295
void writeJobInfo(const std::string &dir) const
Definition: ERF_WriteJobInfo.cpp:9
Here is the call graph for this function:

◆ Write3DPlotFile()

void ERF::Write3DPlotFile ( int  which,
PlotFileType  plotfile_type,
amrex::Vector< std::string >  plot_var_names 
)
308 {
309  auto dPlotTime0 = amrex::second();
310 
311  const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
312  const int ncomp_mf = varnames.size();
313 
314  int ncomp_cons = vars_new[0][Vars::cons].nComp();
315 
316  if (ncomp_mf == 0) return;
317 
318  // We Fillpatch here because some of the derived quantities require derivatives
319  // which require ghost cells to be filled. We do not need to call FillPatcher
320  // because we don't need to set interior fine points.
321  // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
322 
323  // Level 0 FillPatch
325  &vars_new[0][Vars::yvel], &vars_new[0][Vars::zvel]});
326 
327  for (int lev = 1; lev <= finest_level; ++lev) {
328  bool fillset = false;
329  FillPatchFineLevel(lev, t_new[lev], {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::xvel],
330  &vars_new[lev][Vars::yvel], &vars_new[lev][Vars::zvel]},
331  {&vars_new[lev][Vars::cons], &rU_new[lev], &rV_new[lev], &rW_new[lev]},
332  base_state[lev], base_state[lev], fillset);
333  }
334 
335  // Get qmoist pointers if using moisture
336  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
337  for (int lev = 0; lev <= finest_level; ++lev) {
338  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
339  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
340  }
341  }
342 
343  // Vector of MultiFabs for cell-centered data
344  Vector<MultiFab> mf(finest_level+1);
345  for (int lev = 0; lev <= finest_level; ++lev) {
346  mf[lev].define(grids[lev], dmap[lev], ncomp_mf, 0);
347  }
348 
349  // Vector of MultiFabs for nodal data
350  Vector<MultiFab> mf_nd(finest_level+1);
351  if ( SolverChoice::mesh_type != MeshType::ConstantDz) {
352  for (int lev = 0; lev <= finest_level; ++lev) {
353  BoxArray nodal_grids(grids[lev]); nodal_grids.surroundingNodes();
354  mf_nd[lev].define(nodal_grids, dmap[lev], 3, 0);
355  mf_nd[lev].setVal(0.);
356  }
357  }
358 
359  // Vector of MultiFabs for face-centered velocity
360  Vector<MultiFab> mf_u(finest_level+1);
361  Vector<MultiFab> mf_v(finest_level+1);
362  Vector<MultiFab> mf_w(finest_level+1);
363  if (m_plot_face_vels) {
364  for (int lev = 0; lev <= finest_level; ++lev) {
365  BoxArray grid_stag_u(grids[lev]); grid_stag_u.surroundingNodes(0);
366  BoxArray grid_stag_v(grids[lev]); grid_stag_v.surroundingNodes(1);
367  BoxArray grid_stag_w(grids[lev]); grid_stag_w.surroundingNodes(2);
368  mf_u[lev].define(grid_stag_u, dmap[lev], 1, 0);
369  mf_v[lev].define(grid_stag_v, dmap[lev], 1, 0);
370  mf_w[lev].define(grid_stag_w, dmap[lev], 1, 0);
371  MultiFab::Copy(mf_u[lev],vars_new[lev][Vars::xvel],0,0,1,0);
372  MultiFab::Copy(mf_v[lev],vars_new[lev][Vars::yvel],0,0,1,0);
373  MultiFab::Copy(mf_w[lev],vars_new[lev][Vars::zvel],0,0,1,0);
374  }
375  }
376 
377  // Array of MultiFabs for cell-centered velocity
378  Vector<MultiFab> mf_cc_vel(finest_level+1);
379 
380  if (containerHasElement(plot_var_names, "x_velocity" ) ||
381  containerHasElement(plot_var_names, "y_velocity" ) ||
382  containerHasElement(plot_var_names, "z_velocity" ) ||
383  containerHasElement(plot_var_names, "magvel" ) ||
384  containerHasElement(plot_var_names, "vorticity_x") ||
385  containerHasElement(plot_var_names, "vorticity_y") ||
386  containerHasElement(plot_var_names, "vorticity_z") ) {
387 
388  for (int lev = 0; lev <= finest_level; ++lev) {
389  mf_cc_vel[lev].define(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
390  mf_cc_vel[lev].setVal(-1.e20);
391  average_face_to_cellcenter(mf_cc_vel[lev],0,
392  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
393  &vars_new[lev][Vars::yvel],
394  &vars_new[lev][Vars::zvel]});
395  } // lev
396  } // if (vel or vort)
397 
398  // We need ghost cells if computing vorticity
399  if ( containerHasElement(plot_var_names, "vorticity_x")||
400  containerHasElement(plot_var_names, "vorticity_y") ||
401  containerHasElement(plot_var_names, "vorticity_z") )
402  {
403  amrex::Interpolater* mapper = &cell_cons_interp;
404  for (int lev = 1; lev <= finest_level; ++lev)
405  {
406  Vector<MultiFab*> fmf = {&(mf_cc_vel[lev]), &(mf_cc_vel[lev])};
407  Vector<Real> ftime = {t_new[lev], t_new[lev]};
408  Vector<MultiFab*> cmf = {&mf_cc_vel[lev-1], &mf_cc_vel[lev-1]};
409  Vector<Real> ctime = {t_new[lev], t_new[lev]};
410 
411  FillBdyCCVels(mf_cc_vel,lev-1);
412 
413  // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
414  FillPatchTwoLevels(mf_cc_vel[lev], mf_cc_vel[lev].nGrowVect(), IntVect(0,0,0),
415  t_new[lev], cmf, ctime, fmf, ftime,
416  0, 0, mf_cc_vel[lev].nComp(), geom[lev-1], geom[lev],
417  refRatio(lev-1), mapper, domain_bcs_type,
419  } // lev
420  FillBdyCCVels(mf_cc_vel);
421  } // if (vort)
422 
423 
424  for (int lev = 0; lev <= finest_level; ++lev)
425  {
426  // Make sure getPgivenRTh and getTgivenRandRTh don't fail
427  if (check_for_nans) {
429  }
430 
431  int mf_comp = 0;
432 
433  BoxArray ba(vars_new[lev][Vars::cons].boxArray());
434  DistributionMapping dm = vars_new[lev][Vars::cons].DistributionMap();
435 
436  // First, copy any of the conserved state variables into the output plotfile
437  for (int i = 0; i < cons_names.size(); ++i) {
438  if (containerHasElement(plot_var_names, cons_names[i])) {
439  MultiFab::Copy(mf[lev],vars_new[lev][Vars::cons],i,mf_comp,1,0);
440  mf_comp++;
441  }
442  }
443 
444  // Next, check for velocities
445  if (containerHasElement(plot_var_names, "x_velocity")) {
446  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 0, mf_comp, 1, 0);
447  mf_comp += 1;
448  }
449  if (containerHasElement(plot_var_names, "y_velocity")) {
450  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 1, mf_comp, 1, 0);
451  mf_comp += 1;
452  }
453  if (containerHasElement(plot_var_names, "z_velocity")) {
454  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 2, mf_comp, 1, 0);
455  mf_comp += 1;
456  }
457 
458  // Create multifabs for HSE and pressure fields used to derive other quantities
459  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp , 1);
460  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp , 1);
461  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
462 
463  MultiFab pressure;
464 
465  if (solverChoice.anelastic[lev] == 0) {
466  if (containerHasElement(plot_var_names, "pressure") ||
467  containerHasElement(plot_var_names, "pert_pres") ||
468  containerHasElement(plot_var_names, "dpdx") ||
469  containerHasElement(plot_var_names, "dpdy") ||
470  containerHasElement(plot_var_names, "dpdz") ||
471  containerHasElement(plot_var_names, "eq_pot_temp") ||
472  containerHasElement(plot_var_names, "qsat"))
473  {
474  int ng = (containerHasElement(plot_var_names, "dpdx") || containerHasElement(plot_var_names, "dpdy") ||
475  containerHasElement(plot_var_names, "dpdz")) ? 1 : 0;
476 
477  // Allocate space for pressure
478  pressure.define(ba,dm,1,ng);
479 
480  if (ng > 0) {
481  // Default to p_hse as a way of filling ghost cells at domain boundaries
482  MultiFab::Copy(pressure,p_hse,0,0,1,1);
483  }
484 #ifdef _OPENMP
485 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
486 #endif
487  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
488  {
489  const Box& gbx = mfi.growntilebox(IntVect(ng,ng,0));
490 
491  const Array4<Real >& p_arr = pressure.array(mfi);
492  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
493  const int ncomp = vars_new[lev][Vars::cons].nComp();
494 
495  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
496  {
497  Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
498  const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
499  p_arr(i, j, k) = getPgivenRTh(rhotheta,qv_for_p);
500  });
501  } // mfi
502  pressure.FillBoundary(geom[lev].periodicity());
503  } // compute compressible pressure
504  } // not anelastic
505  else {
506  if (containerHasElement(plot_var_names, "dpdx") ||
507  containerHasElement(plot_var_names, "dpdy") ||
508  containerHasElement(plot_var_names, "dpdz") ||
509  containerHasElement(plot_var_names, "eq_pot_temp") ||
510  containerHasElement(plot_var_names, "qsat"))
511  {
512  // Copy p_hse into pressure if using anelastic
513  pressure.define(ba,dm,1,0);
514  MultiFab::Copy(pressure,p_hse,0,0,1,0);
515  }
516  }
517 
518  // Finally, check for any derived quantities and compute them, inserting
519  // them into our output multifab
520  auto calculate_derived = [&](const std::string& der_name,
521  MultiFab& src_mf,
522  decltype(derived::erf_dernull)& der_function)
523  {
524  if (containerHasElement(plot_var_names, der_name)) {
525  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
526 #ifdef _OPENMP
527 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
528 #endif
529  for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
530  {
531  const Box& bx = mfi.tilebox();
532  auto& dfab = dmf[mfi];
533  auto& sfab = src_mf[mfi];
534  der_function(bx, dfab, 0, 1, sfab, Geom(lev), t_new[0], nullptr, lev);
535  }
536 
537  mf_comp++;
538  }
539  };
540 
541  // *****************************************************************************************
542  // NOTE: All derived variables computed below **MUST MATCH THE ORDER** of "derived_names"
543  // defined in ERF.H
544  // *****************************************************************************************
545 
546  calculate_derived("soundspeed", vars_new[lev][Vars::cons], derived::erf_dersoundspeed);
547  if (use_moisture) {
548  calculate_derived("temp", vars_new[lev][Vars::cons], derived::erf_dermoisttemp);
549  } else {
550  calculate_derived("temp", vars_new[lev][Vars::cons], derived::erf_dertemp);
551  }
552  calculate_derived("theta", vars_new[lev][Vars::cons], derived::erf_dertheta);
553  calculate_derived("KE", vars_new[lev][Vars::cons], derived::erf_derKE);
554  calculate_derived("scalar", vars_new[lev][Vars::cons], derived::erf_derscalar);
555  calculate_derived("vorticity_x", mf_cc_vel[lev] , derived::erf_dervortx);
556  calculate_derived("vorticity_y", mf_cc_vel[lev] , derived::erf_dervorty);
557  calculate_derived("vorticity_z", mf_cc_vel[lev] , derived::erf_dervortz);
558  calculate_derived("magvel" , mf_cc_vel[lev] , derived::erf_dermagvel);
559 
560  if (containerHasElement(plot_var_names, "divU"))
561  {
562  // TODO TODO TODO -- we need to convert w to omega here!!
563  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
564  Array<MultiFab const*, AMREX_SPACEDIM> u;
565  u[0] = &(vars_new[lev][Vars::xvel]);
566  u[1] = &(vars_new[lev][Vars::yvel]);
567  u[2] = &(vars_new[lev][Vars::zvel]);
568  compute_divergence (lev, dmf, u, geom[lev]);
569  mf_comp += 1;
570  }
571 
572  if (containerHasElement(plot_var_names, "pres_hse"))
573  {
574  MultiFab::Copy(mf[lev],p_hse,0,mf_comp,1,0);
575  mf_comp += 1;
576  }
577  if (containerHasElement(plot_var_names, "dens_hse"))
578  {
579  MultiFab::Copy(mf[lev],r_hse,0,mf_comp,1,0);
580  mf_comp += 1;
581  }
582  if (containerHasElement(plot_var_names, "theta_hse"))
583  {
584  MultiFab::Copy(mf[lev],th_hse,0,mf_comp,1,0);
585  mf_comp += 1;
586  }
587 
588  if (containerHasElement(plot_var_names, "pressure"))
589  {
590  if (solverChoice.anelastic[lev] == 1) {
591  MultiFab::Copy(mf[lev], p_hse, 0, mf_comp, 1, 0);
592  } else {
593  MultiFab::Copy(mf[lev], pressure, 0, mf_comp, 1, 0);
594  }
595 
596  mf_comp += 1;
597  }
598 
599  if (containerHasElement(plot_var_names, "pert_pres"))
600  {
601  if (solverChoice.anelastic[lev] == 1) {
602  MultiFab::Copy(mf[lev], pp_inc[lev], 0, mf_comp, 1, 0);
603  } else {
604  MultiFab::Copy(mf[lev], pressure, 0, mf_comp, 1, 0);
605  MultiFab::Subtract(mf[lev],p_hse,0,mf_comp,1,IntVect{0});
606  }
607  mf_comp += 1;
608  }
609 
610  if (containerHasElement(plot_var_names, "pert_dens"))
611  {
612 #ifdef _OPENMP
613 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
614 #endif
615  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
616  {
617  const Box& bx = mfi.tilebox();
618  const Array4<Real>& derdat = mf[lev].array(mfi);
619  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
620  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
621  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
622  derdat(i, j, k, mf_comp) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
623  });
624  }
625  mf_comp ++;
626  }
627 
628  if (containerHasElement(plot_var_names, "eq_pot_temp"))
629  {
630 #ifdef _OPENMP
631 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
632 #endif
633  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
634  {
635  const Box& bx = mfi.tilebox();
636  const Array4<Real>& derdat = mf[lev].array(mfi);
637  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
638  const Array4<Real const>& p_arr = pressure.const_array(mfi);
639  const int ncomp = vars_new[lev][Vars::cons].nComp();
640  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
641  Real qv = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
642  Real qc = (use_moisture && (ncomp > RhoQ2_comp)) ? S_arr(i,j,k,RhoQ2_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
643  Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
644  Real fac = Cp_d + Cp_l*(qv + qc);
645  Real pv = erf_esatw(T)*100.0;
646 
647  derdat(i, j, k, mf_comp) = T*std::pow((p_arr(i,j,k) - pv)/p_0, -R_d/fac)*std::exp(L_v*qv/(fac*T)) ;
648  });
649  }
650  mf_comp ++;
651  }
652 
653 #ifdef ERF_USE_WINDFARM
654  if ( containerHasElement(plot_var_names, "num_turb") and
655  (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
656  solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
657  {
658  MultiFab::Copy(mf[lev],Nturb[lev],0,mf_comp,1,0);
659  mf_comp ++;
660  }
661 
662  if ( containerHasElement(plot_var_names, "SMark0") and
663  (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
664  solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
665  {
666  MultiFab::Copy(mf[lev],SMark[lev],0,mf_comp,1,0);
667  mf_comp ++;
668  }
669 
670  if (containerHasElement(plot_var_names, "SMark1") and
671  (solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD))
672  {
673  MultiFab::Copy(mf[lev],SMark[lev],1,mf_comp,1,0);
674  mf_comp ++;
675  }
676 #endif
677 
678  // **********************************************************************************************
679  // Allocate space if we are computing any pressure gradients
680  // **********************************************************************************************
681 
682  Vector<MultiFab> gradp_temp; gradp_temp.resize(AMREX_SPACEDIM);
683  if (containerHasElement(plot_var_names, "dpdx") ||
684  containerHasElement(plot_var_names, "dpdy") ||
685  containerHasElement(plot_var_names, "dpdz") ||
686  containerHasElement(plot_var_names, "pres_hse_x") ||
687  containerHasElement(plot_var_names, "pres_hse_y"))
688  {
689  gradp_temp[GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp_temp[GpVars::gpx].setVal(0.);
690  gradp_temp[GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp_temp[GpVars::gpy].setVal(0.);
691  gradp_temp[GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp_temp[GpVars::gpz].setVal(0.);
692  }
693 
694  // **********************************************************************************************
695  // These are based on computing gradient of full pressure
696  // **********************************************************************************************
697 
698  if (solverChoice.anelastic[lev] == 0) {
699  if ( (containerHasElement(plot_var_names, "dpdx")) ||
700  (containerHasElement(plot_var_names, "dpdy")) ||
701  (containerHasElement(plot_var_names, "dpdz")) ) {
702  compute_gradp(pressure, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
703  get_eb(lev), gradp_temp, solverChoice);
704  }
705  }
706 
707  if (containerHasElement(plot_var_names, "dpdx"))
708  {
709  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
710  {
711  const Box& bx = mfi.tilebox();
712  const Array4<Real >& derdat = mf[lev].array(mfi);
713  const Array4<Real const>& gpx_arr = (solverChoice.anelastic[lev] == 1) ?
714  gradp[lev][GpVars::gpx].array(mfi) : gradp_temp[GpVars::gpx].array(mfi);
715  const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
716  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
717  derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
718  });
719  }
720  mf_comp ++;
721  } // dpdx
722  if (containerHasElement(plot_var_names, "dpdy"))
723  {
724  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
725  {
726  const Box& bx = mfi.tilebox();
727  const Array4<Real >& derdat = mf[lev].array(mfi);
728  const Array4<Real const>& gpy_arr = (solverChoice.anelastic[lev] == 1) ?
729  gradp[lev][GpVars::gpy].array(mfi) : gradp_temp[GpVars::gpy].array(mfi);
730  const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
731  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
732  derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
733  });
734  }
735  mf_comp ++;
736  } // dpdy
737  if (containerHasElement(plot_var_names, "dpdz"))
738  {
739  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
740  {
741  const Box& bx = mfi.tilebox();
742  const Array4<Real >& derdat = mf[lev].array(mfi);
743  const Array4<Real const>& gpz_arr = (solverChoice.anelastic[lev] == 1) ?
744  gradp[lev][GpVars::gpz].array(mfi) : gradp_temp[GpVars::gpz].array(mfi);
745  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
746  derdat(i ,j ,k, mf_comp) = 0.5 * (gpz_arr(i,j,k+1) + gpz_arr(i,j,k));
747  });
748  }
749  mf_comp ++;
750  } // dpdz
751 
752  // **********************************************************************************************
753  // These are based on computing gradient of basestate pressure
754  // **********************************************************************************************
755 
756  if ( (containerHasElement(plot_var_names, "pres_hse_x")) ||
757  (containerHasElement(plot_var_names, "pres_hse_y")) ) {
758  compute_gradp(p_hse, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
759  get_eb(lev), gradp_temp, solverChoice);
760  }
761 
762  if (containerHasElement(plot_var_names, "pres_hse_x"))
763  {
764  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
765  {
766  const Box& bx = mfi.tilebox();
767  const Array4<Real >& derdat = mf[lev].array(mfi);
768  const Array4<Real const>& gpx_arr = gradp_temp[0].array(mfi);
769  const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
770  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
771  derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
772  });
773  }
774  mf_comp += 1;
775  } // pres_hse_x
776 
777  if (containerHasElement(plot_var_names, "pres_hse_y"))
778  {
779  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
780  {
781  const Box& bx = mfi.tilebox();
782  const Array4<Real >& derdat = mf[lev].array(mfi);
783  const Array4<Real const>& gpy_arr = gradp_temp[1].array(mfi);
784  const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
785  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
786  derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
787  });
788  }
789  mf_comp += 1;
790  } // pres_hse_y
791 
792  // **********************************************************************************************
793  // Metric terms
794  // **********************************************************************************************
795 
796  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
797  if (containerHasElement(plot_var_names, "z_phys"))
798  {
799  MultiFab::Copy(mf[lev],*z_phys_cc[lev],0,mf_comp,1,0);
800  mf_comp ++;
801  }
802 
803  if (containerHasElement(plot_var_names, "detJ"))
804  {
805  MultiFab::Copy(mf[lev],*detJ_cc[lev],0,mf_comp,1,0);
806  mf_comp ++;
807  }
808  } // use_terrain
809 
810  if (containerHasElement(plot_var_names, "mapfac")) {
811  amrex::Print() << "You are plotting a 3D version of mapfac; we suggest using the 2D plotfile instead" << std::endl;
812 #ifdef _OPENMP
813 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
814 #endif
815  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
816  {
817  const Box& bx = mfi.tilebox();
818  const Array4<Real>& derdat = mf[lev].array(mfi);
819  const Array4<Real>& mf_m = mapfac[lev][MapFacType::m_x]->array(mfi);
820  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
821  derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
822  });
823  }
824  mf_comp ++;
825  }
826 
827  if (containerHasElement(plot_var_names, "lat_m")) {
828  amrex::Print() << "You are plotting a 3D version of lat_m; we suggest using the 2D plotfile instead" << std::endl;
829  if (lat_m[lev]) {
830 #ifdef _OPENMP
831 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
832 #endif
833  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
834  {
835  const Box& bx = mfi.tilebox();
836  const Array4<Real>& derdat = mf[lev].array(mfi);
837  const Array4<Real>& data = lat_m[lev]->array(mfi);
838  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
839  derdat(i, j, k, mf_comp) = data(i,j,0);
840  });
841  }
842  } else {
843  mf[lev].setVal(0.0,mf_comp,1,0);
844  }
845  mf_comp++;
846  } // lat_m
847 
848  if (containerHasElement(plot_var_names, "lon_m")) {
849  amrex::Print() << "You are plotting a 3D version of lon_m; we suggest using the 2D plotfile instead" << std::endl;
850  if (lon_m[lev]) {
851 #ifdef _OPENMP
852 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
853 #endif
854  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
855  {
856  const Box& bx = mfi.tilebox();
857  const Array4<Real>& derdat = mf[lev].array(mfi);
858  const Array4<Real>& data = lon_m[lev]->array(mfi);
859  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
860  derdat(i, j, k, mf_comp) = data(i,j,0);
861  });
862  }
863  } else {
864  mf[lev].setVal(0.0,mf_comp,1,0);
865  }
866  mf_comp++;
867  } // lon_m
868 
870  if (containerHasElement(plot_var_names, "u_t_avg")) {
871 #ifdef _OPENMP
872 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
873 #endif
874  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
875  {
876  const Box& bx = mfi.tilebox();
877  const Array4<Real>& derdat = mf[lev].array(mfi);
878  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
879  const Real norm = t_avg_cnt[lev];
880  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
881  {
882  derdat(i ,j ,k, mf_comp) = data(i,j,k,0) / norm;
883  });
884  }
885  mf_comp ++;
886  }
887 
888  if (containerHasElement(plot_var_names, "v_t_avg")) {
889 #ifdef _OPENMP
890 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
891 #endif
892  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
893  {
894  const Box& bx = mfi.tilebox();
895  const Array4<Real>& derdat = mf[lev].array(mfi);
896  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
897  const Real norm = t_avg_cnt[lev];
898  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
899  {
900  derdat(i ,j ,k, mf_comp) = data(i,j,k,1) / norm;
901  });
902  }
903  mf_comp ++;
904  }
905 
906  if (containerHasElement(plot_var_names, "w_t_avg")) {
907 #ifdef _OPENMP
908 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
909 #endif
910  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
911  {
912  const Box& bx = mfi.tilebox();
913  const Array4<Real>& derdat = mf[lev].array(mfi);
914  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
915  const Real norm = t_avg_cnt[lev];
916  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
917  {
918  derdat(i ,j ,k, mf_comp) = data(i,j,k,2) / norm;
919  });
920  }
921  mf_comp ++;
922  }
923 
924  if (containerHasElement(plot_var_names, "umag_t_avg")) {
925 #ifdef _OPENMP
926 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
927 #endif
928  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
929  {
930  const Box& bx = mfi.tilebox();
931  const Array4<Real>& derdat = mf[lev].array(mfi);
932  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
933  const Real norm = t_avg_cnt[lev];
934  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
935  {
936  derdat(i ,j ,k, mf_comp) = data(i,j,k,3) / norm;
937  });
938  }
939  mf_comp ++;
940  }
941  }
942 
943  if (containerHasElement(plot_var_names, "nut")) {
944  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
945  MultiFab cmf(vars_new[lev][Vars::cons], make_alias, 0, 1); // to provide rho only
946 #ifdef _OPENMP
947 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
948 #endif
949  for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
950  {
951  const Box& bx = mfi.tilebox();
952  auto prim = dmf[mfi].array();
953  auto const cons = cmf[mfi].const_array();
954  auto const diff = (*eddyDiffs_lev[lev])[mfi].const_array();
955  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
956  {
957  const Real rho = cons(i, j, k, Rho_comp);
958  const Real Kmv = diff(i, j, k, EddyDiff::Mom_v);
959  prim(i,j,k) = Kmv / rho;
960  });
961  }
962 
963  mf_comp++;
964  }
965 
966  if (containerHasElement(plot_var_names, "Kmv")) {
967  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_v,mf_comp,1,0);
968  mf_comp ++;
969  }
970  if (containerHasElement(plot_var_names, "Kmh")) {
971  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_h,mf_comp,1,0);
972  mf_comp ++;
973  }
974  if (containerHasElement(plot_var_names, "Khv")) {
975  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_v,mf_comp,1,0);
976  mf_comp ++;
977  }
978  if (containerHasElement(plot_var_names, "Khh")) {
979  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_h,mf_comp,1,0);
980  mf_comp ++;
981  }
982  if (containerHasElement(plot_var_names, "Lturb")) {
983  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Turb_lengthscale,mf_comp,1,0);
984  mf_comp ++;
985  }
986  if (containerHasElement(plot_var_names, "walldist")) {
987  MultiFab::Copy(mf[lev],*walldist[lev],0,mf_comp,1,0);
988  mf_comp ++;
989  }
990  if (containerHasElement(plot_var_names, "diss")) {
991  MultiFab::Copy(mf[lev],*SFS_diss_lev[lev],0,mf_comp,1,0);
992  mf_comp ++;
993  }
994 
995  // TODO: The size of the q variables can vary with different
996  // moisture models. Therefore, certain components may
997  // reside at different indices. For example, Kessler is
998  // warm but precipitating. This puts qp at index 3.
999  // However, SAM is cold and precipitating so qp is index 4.
1000  // Need to built an external enum struct or a better pathway.
1001 
1002  // NOTE: Protect against accessing non-existent data
1003  if (use_moisture) {
1004  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
1005 
1006  // Moist density
1007  if(containerHasElement(plot_var_names, "moist_density"))
1008  {
1009  int n_start = RhoQ1_comp; // qv
1010  int n_end = RhoQ2_comp; // qc
1011  if (n_qstate_moist > 3) n_end = RhoQ3_comp; // qi
1012  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1013  for (int n_comp(n_start); n_comp <= n_end; ++n_comp) {
1014  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1015  }
1016  mf_comp += 1;
1017  }
1018 
1019  if(containerHasElement(plot_var_names, "qv") && (n_qstate_moist >= 1))
1020  {
1021  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ1_comp, mf_comp, 1, 0);
1022  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1023  mf_comp += 1;
1024  }
1025 
1026  if(containerHasElement(plot_var_names, "qc") && (n_qstate_moist >= 2))
1027  {
1028  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ2_comp, mf_comp, 1, 0);
1029  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1030  mf_comp += 1;
1031  }
1032 
1033  if(containerHasElement(plot_var_names, "qi") && (n_qstate_moist >= 4))
1034  {
1035  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ3_comp, mf_comp, 1, 0);
1036  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1037  mf_comp += 1;
1038  }
1039 
1040  if(containerHasElement(plot_var_names, "qrain") && (n_qstate_moist >= 3))
1041  {
1042  int n_start = (n_qstate_moist > 3) ? RhoQ4_comp : RhoQ3_comp;
1043  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start , mf_comp, 1, 0);
1044  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1045  mf_comp += 1;
1046  }
1047 
1048  if(containerHasElement(plot_var_names, "qsnow") && (n_qstate_moist >= 5))
1049  {
1050  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ5_comp, mf_comp, 1, 0);
1051  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1052  mf_comp += 1;
1053  }
1054 
1055  if(containerHasElement(plot_var_names, "qgraup") && (n_qstate_moist >= 6))
1056  {
1057  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ6_comp, mf_comp, 1, 0);
1058  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1059  mf_comp += 1;
1060  }
1061 
1062  // Precipitating + non-precipitating components
1063  //--------------------------------------------------------------------------
1064  if(containerHasElement(plot_var_names, "qt"))
1065  {
1066  int n_start = RhoQ1_comp; // qv
1067  int n_end = n_start + n_qstate_moist;
1068  MultiFab::Copy(mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1069  for (int n_comp(n_start+1); n_comp < n_end; ++n_comp) {
1070  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1071  }
1072  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1073  mf_comp += 1;
1074  }
1075 
1076  // Non-precipitating components
1077  //--------------------------------------------------------------------------
1078  if (containerHasElement(plot_var_names, "qn"))
1079  {
1080  int n_start = RhoQ1_comp; // qv
1081  int n_end = RhoQ2_comp; // qc
1082  if (n_qstate_moist > 3) n_end = RhoQ3_comp; // qi
1083  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1084  for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
1085  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1086  }
1087  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1088  mf_comp += 1;
1089  }
1090 
1091  // Precipitating components
1092  //--------------------------------------------------------------------------
1093  if(containerHasElement(plot_var_names, "qp") && (n_qstate_moist >= 3))
1094  {
1095  int n_start = (n_qstate_moist > 3) ? RhoQ4_comp : RhoQ3_comp;
1096  int n_end = ncomp_cons - 1;
1097  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1098  for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
1099  MultiFab::Add( mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1100  }
1101  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1102  mf_comp += 1;
1103  }
1104 
1105  if (containerHasElement(plot_var_names, "qsat"))
1106  {
1107 #ifdef _OPENMP
1108 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1109 #endif
1110  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1111  {
1112  const Box& bx = mfi.tilebox();
1113  const Array4<Real>& derdat = mf[lev].array(mfi);
1114  const Array4<Real const>& p_arr = pressure.array(mfi);
1115  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1116  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1117  {
1118  Real qv = S_arr(i,j,k,RhoQ1_comp) / S_arr(i,j,k,Rho_comp);
1119  Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
1120  Real p = p_arr(i,j,k) * Real(0.01);
1121  erf_qsatw(T, p, derdat(i,j,k,mf_comp));
1122  });
1123  }
1124  mf_comp ++;
1125  }
1126 
1127  if ( (solverChoice.moisture_type == MoistureType::Kessler) ||
1128  (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
1129  (solverChoice.moisture_type == MoistureType::SAM_NoIce) )
1130  {
1131  int offset = (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ? 5 : 0;
1132  if (containerHasElement(plot_var_names, "rain_accum"))
1133  {
1134  MultiFab::Copy(mf[lev],*(qmoist[lev][offset]),0,mf_comp,1,0);
1135  mf_comp += 1;
1136  }
1137  }
1138  else if ( (solverChoice.moisture_type == MoistureType::SAM) ||
1139  (solverChoice.moisture_type == MoistureType::Morrison) )
1140  {
1141  int offset = (solverChoice.moisture_type == MoistureType::Morrison) ? 5 : 0;
1142  if (containerHasElement(plot_var_names, "rain_accum"))
1143  {
1144  MultiFab::Copy(mf[lev],*(qmoist[lev][offset]),0,mf_comp,1,0);
1145  mf_comp += 1;
1146  }
1147  if (containerHasElement(plot_var_names, "snow_accum"))
1148  {
1149  MultiFab::Copy(mf[lev],*(qmoist[lev][offset+1]),0,mf_comp,1,0);
1150  mf_comp += 1;
1151  }
1152  if (containerHasElement(plot_var_names, "graup_accum"))
1153  {
1154  MultiFab::Copy(mf[lev],*(qmoist[lev][offset+2]),0,mf_comp,1,0);
1155  mf_comp += 1;
1156  }
1157  }
1158 
1159  if (containerHasElement(plot_var_names, "reflectivity")) {
1160  if (solverChoice.moisture_type == MoistureType::Morrison) {
1161 
1162  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) {
1163  const Box& bx = mfi.tilebox();
1164  const Array4<Real>& derdat = mf[lev].array(mfi);
1165  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1166  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
1167 
1168  Real rho = S_arr(i,j,k,Rho_comp);
1169  Real qv = std::max(0.0,S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp));
1170  Real qpr = std::max(0.0,S_arr(i,j,k,RhoQ4_comp)/S_arr(i,j,k,Rho_comp));
1171  Real qps = std::max(0.0,S_arr(i,j,k,RhoQ5_comp)/S_arr(i,j,k,Rho_comp));
1172  Real qpg = std::max(0.0,S_arr(i,j,k,RhoQ6_comp)/S_arr(i,j,k,Rho_comp));
1173 
1174  Real temp = getTgivenRandRTh(S_arr(i,j,k,Rho_comp),
1175  S_arr(i,j,k,RhoTheta_comp),
1176  qv);
1177  derdat(i, j, k, mf_comp) = compute_max_reflectivity_dbz(rho, temp, qpr, qps, qpg,
1178  1, 1, 1, 1) ;
1179  });
1180  }
1181  mf_comp ++;
1182  }
1183  }
1184 
1185  if (solverChoice.moisture_type == MoistureType::Morrison) {
1186  if (containerHasElement(plot_var_names, "max_reflectivity")) {
1187  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) {
1188  const Box& bx = mfi.tilebox();
1189 
1190  const Array4<Real>& derdat = mf[lev].array(mfi);
1191  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1192 
1193  // collapse to i,j box (ignore vertical for now)
1194  Box b2d = bx;
1195  b2d.setSmall(2,0);
1196  b2d.setBig(2,0);
1197 
1198  ParallelFor(b2d, [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept {
1199 
1200  Real max_dbz = -1.0e30;
1201 
1202  // find max reflectivity over k
1203  for (int k = bx.smallEnd(2); k <= bx.bigEnd(2); ++k) {
1204  Real rho = S_arr(i,j,k,Rho_comp);
1205  Real qv = std::max(0.0, S_arr(i,j,k,RhoQ1_comp)/rho);
1206  Real qpr = std::max(0.0, S_arr(i,j,k,RhoQ4_comp)/rho);
1207  Real qps = std::max(0.0, S_arr(i,j,k,RhoQ5_comp)/rho);
1208  Real qpg = std::max(0.0, S_arr(i,j,k,RhoQ6_comp)/rho);
1209 
1210  Real temp = getTgivenRandRTh(rho, S_arr(i,j,k,RhoTheta_comp), qv);
1211 
1212  Real dbz = compute_max_reflectivity_dbz(rho, temp, qpr, qps, qpg,
1213  1, 1, 1, 1);
1214  max_dbz = amrex::max(max_dbz, dbz);
1215  }
1216 
1217  // store max_dbz into *all* levels for this (i,j)
1218  for (int k = bx.smallEnd(2); k <= bx.bigEnd(2); ++k) {
1219  derdat(i, j, k, mf_comp) = max_dbz;
1220  }
1221  });
1222  }
1223  mf_comp++;
1224  }
1225  }
1226  } // use_moisture
1227 
1228  if (containerHasElement(plot_var_names, "terrain_IB_mask"))
1229  {
1230  MultiFab* terrain_blank = terrain_blanking[lev].get();
1231  MultiFab::Copy(mf[lev],*terrain_blank,0,mf_comp,1,0);
1232  mf_comp ++;
1233  }
1234 
1235  if (containerHasElement(plot_var_names, "volfrac")) {
1236  if ( solverChoice.terrain_type == TerrainType::EB ||
1237  solverChoice.terrain_type == TerrainType::ImmersedForcing)
1238  {
1239  MultiFab::Copy(mf[lev], EBFactory(lev).getVolFrac(), 0, mf_comp, 1, 0);
1240  } else {
1241  mf[lev].setVal(1.0, mf_comp, 1, 0);
1242  }
1243  mf_comp += 1;
1244  }
1245 
1246 #ifdef ERF_COMPUTE_ERROR
1247  // Next, check for error in velocities and if desired, output them -- note we output none or all, not just some
1248  if (containerHasElement(plot_var_names, "xvel_err") ||
1249  containerHasElement(plot_var_names, "yvel_err") ||
1250  containerHasElement(plot_var_names, "zvel_err"))
1251  {
1252  //
1253  // Moving terrain ANALYTICAL
1254  //
1255  Real H = geom[lev].ProbHi()[2];
1256  Real Ampl = 0.16;
1257  Real wavelength = 100.;
1258  Real kp = 2. * PI / wavelength;
1259  Real g = CONST_GRAV;
1260  Real omega = std::sqrt(g * kp);
1261  Real omega_t = omega * t_new[lev];
1262 
1263  const auto dx = geom[lev].CellSizeArray();
1264 
1265 #ifdef _OPENMP
1266 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1267 #endif
1268  for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
1269  {
1270  const Box& bx = mfi.validbox();
1271  Box xbx(bx); xbx.surroundingNodes(0);
1272  const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
1273  const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
1274 
1275  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1276 
1277  ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1278  {
1279  Real x = i * dx[0];
1280  Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
1281 
1282  Real z_base = Ampl * std::sin(kp * x - omega_t);
1283  z -= z_base;
1284 
1285  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1286 
1287  xvel_arr(i,j,k) -= -Ampl * omega * fac * std::sin(kp * x - omega_t);
1288  });
1289 
1290  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1291  {
1292  Real x = (i + 0.5) * dx[0];
1293  Real z = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
1294 
1295  Real z_base = Ampl * std::sin(kp * x - omega_t);
1296  z -= z_base;
1297 
1298  Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
1299 
1300  zvel_arr(i,j,k) -= Ampl * omega * fac * std::cos(kp * x - omega_t);
1301  });
1302  }
1303 
1304  MultiFab temp_mf(mf[lev].boxArray(), mf[lev].DistributionMap(), AMREX_SPACEDIM, 0);
1305  average_face_to_cellcenter(temp_mf,0,
1306  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
1307 
1308  if (containerHasElement(plot_var_names, "xvel_err")) {
1309  MultiFab::Copy(mf[lev],temp_mf,0,mf_comp,1,0);
1310  mf_comp += 1;
1311  }
1312  if (containerHasElement(plot_var_names, "yvel_err")) {
1313  MultiFab::Copy(mf[lev],temp_mf,1,mf_comp,1,0);
1314  mf_comp += 1;
1315  }
1316  if (containerHasElement(plot_var_names, "zvel_err")) {
1317  MultiFab::Copy(mf[lev],temp_mf,2,mf_comp,1,0);
1318  mf_comp += 1;
1319  }
1320 
1321  // Now restore the velocities to what they were
1322 #ifdef _OPENMP
1323 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1324 #endif
1325  for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
1326  {
1327  const Box& bx = mfi.validbox();
1328  Box xbx(bx); xbx.surroundingNodes(0);
1329 
1330  const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
1331  const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
1332 
1333  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1334 
1335  ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1336  {
1337  Real x = i * dx[0];
1338  Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
1339  Real z_base = Ampl * std::sin(kp * x - omega_t);
1340 
1341  z -= z_base;
1342 
1343  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1344  xvel_arr(i,j,k) += -Ampl * omega * fac * std::sin(kp * x - omega_t);
1345  });
1346  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1347  {
1348  Real x = (i + 0.5) * dx[0];
1349  Real z = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
1350  Real z_base = Ampl * std::sin(kp * x - omega_t);
1351 
1352  z -= z_base;
1353  Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
1354 
1355  zvel_arr(i,j,k) += Ampl * omega * fac * std::cos(kp * x - omega_t);
1356  });
1357  }
1358  } // end xvel_err, yvel_err, zvel_err
1359 
1360  if (containerHasElement(plot_var_names, "pp_err"))
1361  {
1362  // Moving terrain ANALYTICAL
1363 #ifdef _OPENMP
1364 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1365 #endif
1366  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1367  {
1368  const Box& bx = mfi.tilebox();
1369  const Array4<Real>& derdat = mf[lev].array(mfi);
1370  const Array4<Real const>& p0_arr = p_hse.const_array(mfi);
1371  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1372 
1373  const auto dx = geom[lev].CellSizeArray();
1374  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1375  const Array4<Real const>& p_arr = pressure.const_array(mfi);
1376  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
1377 
1378  Real H = geom[lev].ProbHi()[2];
1379  Real Ampl = 0.16;
1380  Real wavelength = 100.;
1381  Real kp = 2. * PI / wavelength;
1382  Real g = CONST_GRAV;
1383  Real omega = std::sqrt(g * kp);
1384  Real omega_t = omega * t_new[lev];
1385 
1386  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1387  {
1388  derdat(i, j, k, mf_comp) = p_arr(i,j,k) - p0_arr(i,j,k);
1389 
1390  Real rho_hse = r0_arr(i,j,k);
1391 
1392  Real x = (i + 0.5) * dx[0];
1393  Real z = 0.125 * ( z_nd(i,j,k ) + z_nd(i+1,j,k ) + z_nd(i,j+1,k ) + z_nd(i+1,j+1,k )
1394  +z_nd(i,j,k+1) + z_nd(i+1,j,k+1) + z_nd(i,j+1,k+1) + z_nd(i+1,j+1,k+1) );
1395  Real z_base = Ampl * std::sin(kp * x - omega_t);
1396 
1397  z -= z_base;
1398  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1399  Real pprime_exact = -(Ampl * omega * omega / kp) * fac *
1400  std::sin(kp * x - omega_t) * r0_arr(i,j,k);
1401 
1402  derdat(i,j,k,mf_comp) -= pprime_exact;
1403  });
1404  }
1405  mf_comp += 1;
1406  }
1407 #endif
1408 
1409  if (solverChoice.rad_type != RadiationType::None) {
1410  if (containerHasElement(plot_var_names, "qsrc_sw")) {
1411  MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 0, mf_comp, 1, 0);
1412  mf_comp += 1;
1413  }
1414  if (containerHasElement(plot_var_names, "qsrc_lw")) {
1415  MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 1, mf_comp, 1, 0);
1416  mf_comp += 1;
1417  }
1418  }
1419 
1420  // *****************************************************************************************
1421  // End of derived variables corresponding to "derived_names" in ERF.H
1422  //
1423  // Particles and microphysics can provide additional outputs, which are handled below.
1424  // *****************************************************************************************
1425 
1426 #ifdef ERF_USE_PARTICLES
1427  const auto& particles_namelist( particleData.getNames() );
1428 
1429  if (containerHasElement(plot_var_names, "tracer_particles_count")) {
1430  if (particles_namelist.size() == 0) {
1431  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
1432  temp_dat.setVal(0);
1433  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1434  mf_comp += 1;
1435  } else {
1436  for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++) {
1437  if (containerHasElement(plot_var_names, std::string(particles_namelist[i]+"_count"))) {
1438  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
1439  temp_dat.setVal(0);
1440  if (particleData.HasSpecies(particles_namelist[i])) {
1441  particleData[particles_namelist[i]]->Increment(temp_dat, lev);
1442  }
1443  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1444  mf_comp += 1;
1445  }
1446  }
1447  }
1448  }
1449 
1450  Vector<std::string> particle_mesh_plot_names(0);
1451  particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
1452 
1453  for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
1454  std::string plot_var_name(particle_mesh_plot_names[i]);
1455  if (containerHasElement(plot_var_names, plot_var_name) ) {
1456  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
1457  temp_dat.setVal(0);
1458  particleData.GetMeshPlotVar(plot_var_name, temp_dat, lev);
1459  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1460  mf_comp += 1;
1461  }
1462  }
1463 #endif
1464 
1465  {
1466  Vector<std::string> microphysics_plot_names;
1467  micro->GetPlotVarNames(microphysics_plot_names);
1468  for (auto& plot_name : microphysics_plot_names) {
1469  if (containerHasElement(plot_var_names, plot_name)) {
1470  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
1471  temp_dat.setVal(0);
1472  micro->GetPlotVar(plot_name, temp_dat, lev);
1473  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1474  mf_comp += 1;
1475  }
1476  }
1477  }
1478 
1479 
1480  }
1481 
1482  if (solverChoice.terrain_type == TerrainType::EB)
1483  {
1484  for (int lev = 0; lev <= finest_level; ++lev) {
1485  EB_set_covered(mf[lev], 0.0);
1486  }
1487  }
1488 
1489  // Fill terrain distortion MF (nu_nd)
1490  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1491  for (int lev(0); lev <= finest_level; ++lev) {
1492  MultiFab::Copy(mf_nd[lev],*z_phys_nd[lev],0,2,1,0);
1493  Real dz = Geom()[lev].CellSizeArray()[2];
1494  for (MFIter mfi(mf_nd[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
1495  const Box& bx = mfi.tilebox();
1496  Array4<Real> mf_arr = mf_nd[lev].array(mfi);
1497  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1498  {
1499  mf_arr(i,j,k,2) -= k * dz;
1500  });
1501  }
1502  }
1503  }
1504 
1505  std::string plotfilename;
1506  std::string plotfilenameU;
1507  std::string plotfilenameV;
1508  std::string plotfilenameW;
1509  if (which == 1) {
1510  plotfilename = Concatenate(plot3d_file_1, istep[0], 5);
1511  plotfilenameU = Concatenate(plot3d_file_1+"U", istep[0], 5);
1512  plotfilenameV = Concatenate(plot3d_file_1+"V", istep[0], 5);
1513  plotfilenameW = Concatenate(plot3d_file_1+"W", istep[0], 5);
1514  } else if (which == 2) {
1515  plotfilename = Concatenate(plot3d_file_2, istep[0], 5);
1516  plotfilenameU = Concatenate(plot3d_file_2+"U", istep[0], 5);
1517  plotfilenameV = Concatenate(plot3d_file_2+"V", istep[0], 5);
1518  plotfilenameW = Concatenate(plot3d_file_2+"W", istep[0], 5);
1519  }
1520 
1521  // LSM writes it's own data
1522  if (which==1 && plot_lsm) {
1523  lsm.Plot_Lsm_Data(t_new[0], istep, refRatio());
1524  }
1525 
1526 #ifdef ERF_USE_RRTMGP
1527  /*
1528  // write additional RRTMGP data
1529  // TODO: currently single level only
1530  if (which==1 && plot_rad) {
1531  rad[0]->writePlotfile(plot_file_1, t_new[0], istep[0]);
1532  }
1533  */
1534 #endif
1535 
1536  // Single level
1537  if (finest_level == 0)
1538  {
1539  if (plotfile_type == PlotFileType::Amrex)
1540  {
1541  Print() << "Writing native 3D plotfile " << plotfilename << "\n";
1542  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1543  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1544  GetVecOfConstPtrs(mf),
1545  GetVecOfConstPtrs(mf_nd),
1546  varnames,
1547  Geom(), t_new[0], istep, refRatio());
1548  } else {
1549  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1550  GetVecOfConstPtrs(mf),
1551  varnames,
1552  Geom(), t_new[0], istep, refRatio());
1553  }
1554  writeJobInfo(plotfilename);
1555 
1556  if (m_plot_face_vels) {
1557  Print() << "Writing face velocities" << std::endl;
1558  WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
1559  GetVecOfConstPtrs(mf_u),
1560  {"x_velocity_stag"},
1561  Geom(), t_new[0], istep, refRatio());
1562  WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
1563  GetVecOfConstPtrs(mf_v),
1564  {"y_velocity_stag"},
1565  Geom(), t_new[0], istep, refRatio());
1566  WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
1567  GetVecOfConstPtrs(mf_w),
1568  {"z_velocity_stag"},
1569  Geom(), t_new[0], istep, refRatio());
1570  }
1571 
1572 #ifdef ERF_USE_PARTICLES
1573  particleData.writePlotFile(plotfilename);
1574 #endif
1575 #ifdef ERF_USE_NETCDF
1576  } else if (plotfile_type == PlotFileType::Netcdf) {
1577  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::StaticFittedMesh);
1578  int lev = 0;
1579  int l_which = 0;
1580  const Real* p_lo = geom[lev].ProbLo();
1581  const Real* p_hi = geom[lev].ProbHi();
1582  const auto dx = geom[lev].CellSize();
1583  writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
1584  {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
1585  geom[lev].Domain(), t_new[0], start_bdy_time);
1586 #endif
1587  } else {
1588  // Here we assume the plotfile_type is PlotFileType::None
1589  Print() << "Writing no 3D plotfile since plotfile_type is none" << std::endl;
1590  }
1591 
1592  } else { // Multilevel
1593 
1594  if (plotfile_type == PlotFileType::Amrex) {
1595 
1596  int lev0 = 0;
1597  int desired_ratio = std::max(std::max(ref_ratio[lev0][0],ref_ratio[lev0][1]),ref_ratio[lev0][2]);
1598  bool any_ratio_one = ( ( (ref_ratio[lev0][0] == 1) || (ref_ratio[lev0][1] == 1) ) ||
1599  (ref_ratio[lev0][2] == 1) );
1600  for (int lev = 1; lev < finest_level; lev++) {
1601  any_ratio_one = any_ratio_one ||
1602  ( ( (ref_ratio[lev][0] == 1) || (ref_ratio[lev][1] == 1) ) ||
1603  (ref_ratio[lev][2] == 1) );
1604  }
1605 
1606  if (any_ratio_one && m_expand_plotvars_to_unif_rr)
1607  {
1608  Vector<IntVect> r2(finest_level);
1609  Vector<Geometry> g2(finest_level+1);
1610  Vector<MultiFab> mf2(finest_level+1);
1611 
1612  mf2[0].define(grids[0], dmap[0], ncomp_mf, 0);
1613 
1614  // Copy level 0 as is
1615  MultiFab::Copy(mf2[0],mf[0],0,0,mf[0].nComp(),0);
1616 
1617  // Define a new multi-level array of Geometry's so that we pass the new "domain" at lev > 0
1618  Array<int,AMREX_SPACEDIM> periodicity =
1619  {Geom()[lev0].isPeriodic(0),Geom()[lev0].isPeriodic(1),Geom()[lev0].isPeriodic(2)};
1620  g2[lev0].define(Geom()[lev0].Domain(),&(Geom()[lev0].ProbDomain()),0,periodicity.data());
1621 
1622  r2[0] = IntVect(desired_ratio/ref_ratio[lev0][0],
1623  desired_ratio/ref_ratio[lev0][1],
1624  desired_ratio/ref_ratio[lev0][2]);
1625 
1626  for (int lev = 1; lev <= finest_level; ++lev) {
1627  if (lev > 1) {
1628  r2[lev-1][0] = r2[lev-2][0] * desired_ratio / ref_ratio[lev-1][0];
1629  r2[lev-1][1] = r2[lev-2][1] * desired_ratio / ref_ratio[lev-1][1];
1630  r2[lev-1][2] = r2[lev-2][2] * desired_ratio / ref_ratio[lev-1][2];
1631  }
1632 
1633  mf2[lev].define(refine(grids[lev],r2[lev-1]), dmap[lev], ncomp_mf, 0);
1634 
1635  // Set the new problem domain
1636  Box d2(Geom()[lev].Domain());
1637  d2.refine(r2[lev-1]);
1638 
1639  g2[lev].define(d2,&(Geom()[lev].ProbDomain()),0,periodicity.data());
1640  }
1641 
1642  //
1643  // We need to make a temporary that is the size of ncomp_mf
1644  // in order to not get an out of bounds error
1645  // even though the values will not be used
1646  //
1647  Vector<BCRec> temp_domain_bcs_type;
1648  temp_domain_bcs_type.resize(ncomp_mf);
1649 
1650  //
1651  // Do piecewise constant interpolation of mf into mf2
1652  //
1653  for (int lev = 1; lev <= finest_level; ++lev) {
1654  Interpolater* mapper_c = &pc_interp;
1655  InterpFromCoarseLevel(mf2[lev], t_new[lev], mf[lev],
1656  0, 0, ncomp_mf,
1657  geom[lev], g2[lev],
1659  r2[lev-1], mapper_c, temp_domain_bcs_type, 0);
1660  }
1661 
1662  // Define an effective ref_ratio which is isotropic to be passed into WriteMultiLevelPlotfile
1663  Vector<IntVect> rr(finest_level);
1664  for (int lev = 0; lev < finest_level; ++lev) {
1665  rr[lev] = IntVect(desired_ratio);
1666  }
1667 
1668  Print() << "Writing 3D plotfile " << plotfilename << "\n";
1669  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1670  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1671  GetVecOfConstPtrs(mf2),
1672  GetVecOfConstPtrs(mf_nd),
1673  varnames,
1674  g2, t_new[0], istep, rr);
1675  } else {
1676  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1677  GetVecOfConstPtrs(mf2), varnames,
1678  g2, t_new[0], istep, rr);
1679  }
1680 
1681  } else {
1682  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1683  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1684  GetVecOfConstPtrs(mf),
1685  GetVecOfConstPtrs(mf_nd),
1686  varnames,
1687  geom, t_new[0], istep, ref_ratio);
1688  } else {
1689  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1690  GetVecOfConstPtrs(mf), varnames,
1691  geom, t_new[0], istep, ref_ratio);
1692  }
1693  if (m_plot_face_vels) {
1694  Print() << "Writing face velocities" << std::endl;
1695  WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
1696  GetVecOfConstPtrs(mf_u),
1697  {"x_velocity_stag"},
1698  geom, t_new[0], istep, ref_ratio);
1699  WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
1700  GetVecOfConstPtrs(mf_v),
1701  {"y_velocity_stag"},
1702  geom, t_new[0], istep, ref_ratio);
1703  WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
1704  GetVecOfConstPtrs(mf_w),
1705  {"z_velocity_stag"},
1706  geom, t_new[0], istep, ref_ratio);
1707  }
1708  } // ref_ratio test
1709 
1710  writeJobInfo(plotfilename);
1711 
1712 #ifdef ERF_USE_PARTICLES
1713  particleData.writePlotFile(plotfilename);
1714 #endif
1715 
1716 #ifdef ERF_USE_NETCDF
1717  } else if (plotfile_type == PlotFileType::Netcdf) {
1718  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::StaticFittedMesh);
1719  for (int lev = 0; lev <= finest_level; ++lev) {
1720  for (int which_box = 0; which_box < num_boxes_at_level[lev]; which_box++) {
1721  Box bounding_region = (lev == 0) ? geom[lev].Domain() : boxes_at_level[lev][which_box];
1722  const Real* p_lo = geom[lev].ProbLo();
1723  const Real* p_hi = geom[lev].ProbHi();
1724  const auto dx = geom[lev].CellSizeArray();
1725  writeNCPlotFile(lev, which_box, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
1726  {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
1727  bounding_region, t_new[0], start_bdy_time);
1728  }
1729  }
1730 #endif
1731  }
1732  } // end multi-level
1733 
1734  if (verbose > 0)
1735  {
1736  auto dPlotTime = amrex::second() - dPlotTime0;
1737  ParallelDescriptor::ReduceRealMax(dPlotTime,ParallelDescriptor::IOProcessorNumber());
1738  amrex::Print() << "3DPlotfile write time = " << dPlotTime << " seconds." << '\n';
1739  }
1740 }
constexpr amrex::Real PI
Definition: ERF_Constants.H:6
constexpr amrex::Real Cp_l
Definition: ERF_Constants.H:14
#define RhoQ4_comp
Definition: ERF_IndexDefines.H:45
void compute_gradp(const MultiFab &p, const Geometry &geom, const MultiFab &z_phys_nd, const MultiFab &z_phys_cc, Vector< std::unique_ptr< MultiFab >> &mapfac, const eb_ &ebfact, Vector< MultiFab > &gradp, const SolverChoice &solverChoice)
Definition: ERF_MakeGradP.cpp:81
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real erf_esatw(amrex::Real t)
Definition: ERF_MicrophysicsUtils.H:68
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void erf_qsatw(amrex::Real t, amrex::Real p, amrex::Real &qsatw)
Definition: ERF_MicrophysicsUtils.H:166
PhysBCFunctNoOp null_bc_for_fill
Definition: ERF_Plotfile.cpp:8
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real compute_max_reflectivity_dbz(amrex::Real rho_air, amrex::Real tmk, amrex::Real qra, amrex::Real qsn, amrex::Real qgr, int in0r, int in0s, int in0g, int iliqskin)
Definition: ERF_StormDiagnostics.H:13
void WriteMultiLevelPlotfileWithTerrain(const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
Definition: ERF_Plotfile.cpp:1743
void Plot_Lsm_Data(amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &ref_ratio)
Definition: ERF_LandSurface.H:119
@ Turb_lengthscale
Definition: ERF_IndexDefines.H:180
@ Mom_h
Definition: ERF_IndexDefines.H:170
@ Theta_h
Definition: ERF_IndexDefines.H:171
@ qpg
Definition: ERF_Morrison.H:41
@ qps
Definition: ERF_Morrison.H:40
@ qpr
Definition: ERF_Morrison.H:39
void erf_dervortx(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:200
void erf_dervorty(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:228
void erf_dermagvel(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:319
void erf_dernull(const Box &, FArrayBox &, int, int, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:39
void erf_dertemp(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:91
void erf_derKE(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:186
void erf_dermoisttemp(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:113
void erf_dersoundspeed(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:58
real(c_double), parameter g
Definition: ERF_module_model_constants.F90:19
Here is the call graph for this function:

◆ write_1D_profiles()

void ERF::write_1D_profiles ( amrex::Real  time)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Parameters
timeCurrent time
18 {
19  BL_PROFILE("ERF::write_1D_profiles()");
20 
21  if (NumDataLogs() > 1)
22  {
23  // Define the 1d arrays we will need
24  Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
25  Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
26  Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
27  Gpu::HostVector<Real> h_avg_wthv;
28  Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
29  Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
30  Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
31  Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
32  Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
33  Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
34 
35  if (NumDataLogs() > 1) {
37  h_avg_u, h_avg_v, h_avg_w,
38  h_avg_rho, h_avg_th, h_avg_ksgs,
39  h_avg_Kmv, h_avg_Khv,
40  h_avg_qv, h_avg_qc, h_avg_qr,
41  h_avg_wqv, h_avg_wqc, h_avg_wqr,
42  h_avg_qi, h_avg_qs, h_avg_qg,
43  h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
44  h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
45  h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
46  h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
47  h_avg_wthv);
48  }
49 
50  if (NumDataLogs() > 3 && time > 0.) {
51  derive_stress_profiles(h_avg_tau11, h_avg_tau12, h_avg_tau13,
52  h_avg_tau22, h_avg_tau23, h_avg_tau33,
53  h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
54  h_avg_sgsdiss);
55  }
56 
57  int hu_size = h_avg_u.size();
58 
59  auto const& dx = geom[0].CellSizeArray();
60  if (ParallelDescriptor::IOProcessor()) {
61  if (NumDataLogs() > 1) {
62  std::ostream& data_log1 = DataLog(1);
63  if (data_log1.good()) {
64  // Write the quantities at this time
65  for (int k = 0; k < hu_size; k++) {
66  Real z;
67  if (zlevels_stag[0].size() > 1) {
68  z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
69  } else {
70  z = (k + 0.5)* dx[2];
71  }
72  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
73  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
74  << h_avg_u[k] << " " << h_avg_v[k] << " " << h_avg_w[k] << " "
75  << h_avg_rho[k] << " " << h_avg_th[k] << " " << h_avg_ksgs[k] << " "
76  << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
77  << h_avg_qv[k] << " " << h_avg_qc[k] << " " << h_avg_qr[k] << " "
78  << h_avg_qi[k] << " " << h_avg_qs[k] << " " << h_avg_qg[k]
79  << std::endl;
80  } // loop over z
81  } // if good
82  } // NumDataLogs
83 
84  if (NumDataLogs() > 2) {
85  std::ostream& data_log2 = DataLog(2);
86  if (data_log2.good()) {
87  // Write the perturbational quantities at this time
88  for (int k = 0; k < hu_size; k++) {
89  Real z;
90  if (zlevels_stag[0].size() > 1) {
91  z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
92  } else {
93  z = (k + 0.5)* dx[2];
94  }
95  Real thv = h_avg_th[k] * (1 + 0.61*h_avg_qv[k] - h_avg_qc[k] - h_avg_qr[k]);
96  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
97  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
98  << h_avg_uu[k] - h_avg_u[k]*h_avg_u[k] << " "
99  << h_avg_uv[k] - h_avg_u[k]*h_avg_v[k] << " "
100  << h_avg_uw[k] - h_avg_u[k]*h_avg_w[k] << " "
101  << h_avg_vv[k] - h_avg_v[k]*h_avg_v[k] << " "
102  << h_avg_vw[k] - h_avg_v[k]*h_avg_w[k] << " "
103  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " "
104  << h_avg_uth[k] - h_avg_u[k]*h_avg_th[k] << " "
105  << h_avg_vth[k] - h_avg_v[k]*h_avg_th[k] << " "
106  << h_avg_wth[k] - h_avg_w[k]*h_avg_th[k] << " "
107  << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " "
108  // Note: <u'_i u'_i u'_j> = <u_i u_i u_j>
109  // - <u_i u_i> * <u_j>
110  // - 2*<u_i> * <u_i u_j>
111  // + 2*<u_i>*<u_i> * <u_j>
112  << h_avg_uiuiu[k]
113  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_u[k]
114  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + h_avg_w[k]*h_avg_uw[k])
115  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_u[k]
116  << " " // (u'_i u'_i)u'
117  << h_avg_uiuiv[k]
118  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_v[k]
119  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + h_avg_w[k]*h_avg_vw[k])
120  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_v[k]
121  << " " // (u'_i u'_i)v'
122  << h_avg_uiuiw[k]
123  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_w[k]
124  - 2*(h_avg_u[k]*h_avg_uw[k] + h_avg_v[k]*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
125  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
126  << " " // (u'_i u'_i)w'
127  << h_avg_pu[k] - h_avg_p[k]*h_avg_u[k] << " "
128  << h_avg_pv[k] - h_avg_p[k]*h_avg_v[k] << " "
129  << h_avg_pw[k] - h_avg_p[k]*h_avg_w[k] << " "
130  << h_avg_wqv[k] - h_avg_qv[k]*h_avg_w[k] << " "
131  << h_avg_wqc[k] - h_avg_qc[k]*h_avg_w[k] << " "
132  << h_avg_wqr[k] - h_avg_qr[k]*h_avg_w[k] << " "
133  << h_avg_wthv[k] - h_avg_w[k]*thv
134  << std::endl;
135  } // loop over z
136  } // if good
137  } // NumDataLogs
138 
139  if (NumDataLogs() > 3 && time > 0.) {
140  std::ostream& data_log3 = DataLog(3);
141  if (data_log3.good()) {
142  // Write the average stresses
143  for (int k = 0; k < hu_size; k++) {
144  Real z;
145  if (zlevels_stag[0].size() > 1) {
146  z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
147  } else {
148  z = (k + 0.5)* dx[2];
149  }
150  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
151  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
152  << h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
153  << h_avg_tau22[k] << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
154  << h_avg_sgshfx[k] << " "
155  << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
156  << h_avg_sgsdiss[k]
157  << std::endl;
158  } // loop over z
159  } // if good
160  } // if (NumDataLogs() > 3)
161  } // if IOProcessor
162  } // if (NumDataLogs() > 1)
163 }
void derive_diag_profiles(amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
Definition: ERF_Write1DProfiles.cpp:190
void derive_stress_profiles(amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
Definition: ERF_Write1DProfiles.cpp:475

◆ write_1D_profiles_stag()

void ERF::write_1D_profiles_stag ( amrex::Real  time)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Quantities are output at their native grid locations. Therefore, w and associated flux quantities <(•)'w'>, tau13, and tau23 (where '•' includes u, v, p, theta, ...) will be output at staggered heights (i.e., coincident with z faces) rather than cell-center heights to avoid performing additional averaging. Unstaggered (i.e., cell-centered) quantities are output alongside staggered quantities at the lower cell faces in the log file; these quantities will have a zero value at the big end, corresponding to k=Nz+1.

The structure of file should follow ERF_Write1DProfiles.cpp

Parameters
timeCurrent time
26 {
27  BL_PROFILE("ERF::write_1D_profiles()");
28 
29  if (NumDataLogs() > 1)
30  {
31  // Define the 1d arrays we will need
32  Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
33  Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
34  Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
35  Gpu::HostVector<Real> h_avg_wthv;
36  Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
37  Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
38  Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
39  Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
40  Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
41  Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
42 
43  if (NumDataLogs() > 1) {
45  h_avg_u, h_avg_v, h_avg_w,
46  h_avg_rho, h_avg_th, h_avg_ksgs,
47  h_avg_Kmv, h_avg_Khv,
48  h_avg_qv, h_avg_qc, h_avg_qr,
49  h_avg_wqv, h_avg_wqc, h_avg_wqr,
50  h_avg_qi, h_avg_qs, h_avg_qg,
51  h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
52  h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
53  h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
54  h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
55  h_avg_wthv);
56  }
57 
58  if (NumDataLogs() > 3 && time > 0.) {
59  derive_stress_profiles_stag(h_avg_tau11, h_avg_tau12, h_avg_tau13,
60  h_avg_tau22, h_avg_tau23, h_avg_tau33,
61  h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
62  h_avg_sgsdiss);
63  }
64 
65  int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
66 
67  auto const& dx = geom[0].CellSizeArray();
68  if (ParallelDescriptor::IOProcessor()) {
69  if (NumDataLogs() > 1) {
70  std::ostream& data_log1 = DataLog(1);
71  if (data_log1.good()) {
72  // Write the quantities at this time
73  for (int k = 0; k < unstag_size; k++) {
74  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
75  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
76  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
77  << h_avg_u[k] << " " << h_avg_v[k] << " " << h_avg_w[k] << " "
78  << h_avg_rho[k] << " " << h_avg_th[k] << " " << h_avg_ksgs[k] << " "
79  << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
80  << h_avg_qv[k] << " " << h_avg_qc[k] << " " << h_avg_qr[k] << " "
81  << h_avg_qi[k] << " " << h_avg_qs[k] << " " << h_avg_qg[k]
82  << std::endl;
83  } // loop over z
84  // Write top face values
85  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
86  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
87  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
88  << 0 << " " << 0 << " " << h_avg_w[unstag_size] << " "
89  << 0 << " " << 0 << " " << 0 << " " // rho, theta, ksgs
90  << 0 << " " << 0 << " " // Kmv, Khv
91  << 0 << " " << 0 << " " << 0 << " " // qv, qc, qr
92  << 0 << " " << 0 << " " << 0 // qi, qs, qg
93  << std::endl;
94  } // if good
95  } // NumDataLogs
96 
97  if (NumDataLogs() > 2) {
98  std::ostream& data_log2 = DataLog(2);
99  if (data_log2.good()) {
100  // Write the perturbational quantities at this time
101  // For surface values (k=0), assume w = uw = vw = ww = 0
102  Real w_cc = h_avg_w[1] / 2; // w at first cell center
103  Real uw_cc = h_avg_uw[1] / 2; // u*w at first cell center
104  Real vw_cc = h_avg_vw[1] / 2; // v*w at first cell center
105  Real ww_cc = h_avg_ww[1] / 2; // w*w at first cell center
106  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
107  << std::setw(datwidth) << std::setprecision(datprecision) << 0 << " "
108  << h_avg_uu[0] - h_avg_u[0]*h_avg_u[0] << " " // u'u'
109  << h_avg_uv[0] - h_avg_u[0]*h_avg_v[0] << " " // u'v'
110  << 0 << " " // u'w'
111  << h_avg_vv[0] - h_avg_v[0]*h_avg_v[0] << " " // v'v'
112  << 0 << " " // v'w'
113  << 0 << " " // w'w'
114  << h_avg_uth[0] - h_avg_u[0]*h_avg_th[0] << " " // u'th'
115  << h_avg_vth[0] - h_avg_v[0]*h_avg_th[0] << " " // v'th'
116  << 0 << " " // w'th'
117  << h_avg_thth[0] - h_avg_th[0]*h_avg_th[0] << " " // th'th'
118  << h_avg_uiuiu[0]
119  - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_u[0]
120  - 2*(h_avg_u[0]*h_avg_uu[0] + h_avg_v[0]*h_avg_uv[0] + w_cc*uw_cc)
121  + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_u[0]
122  << " " // (u'_i u'_i)u'
123  << h_avg_uiuiv[0]
124  - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_v[0]
125  - 2*(h_avg_u[0]*h_avg_uv[0] + h_avg_v[0]*h_avg_vv[0] + w_cc*vw_cc)
126  + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_v[0]
127  << " " // (u'_i u'_i)v'
128  << 0 << " " // (u'_i u'_i)w'
129  << h_avg_pu[0] - h_avg_p[0]*h_avg_u[0] << " " // p'u'
130  << h_avg_pv[0] - h_avg_p[0]*h_avg_v[0] << " " // p'v'
131  << 0 << " " // p'w'
132  << 0 << " " // qv'w'
133  << 0 << " " // qc'w'
134  << 0 << " " // qr'w'
135  << 0 // thv'w'
136  << std::endl;
137 
138  // For internal values, interpolate scalar quantities to faces
139  for (int k = 1; k < unstag_size; k++) {
140  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
141  Real uface = 0.5*(h_avg_u[k] + h_avg_u[k-1]);
142  Real vface = 0.5*(h_avg_v[k] + h_avg_v[k-1]);
143  Real thface = 0.5*(h_avg_th[k] + h_avg_th[k-1]);
144  Real pface = 0.5*(h_avg_p[k] + h_avg_p[k-1]);
145  Real qvface = 0.5*(h_avg_qv[k] + h_avg_qv[k-1]);
146  Real qcface = 0.5*(h_avg_qc[k] + h_avg_qc[k-1]);
147  Real qrface = 0.5*(h_avg_qr[k] + h_avg_qr[k-1]);
148  Real uuface = 0.5*(h_avg_uu[k] + h_avg_uu[k-1]);
149  Real vvface = 0.5*(h_avg_vv[k] + h_avg_vv[k-1]);
150  Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
151  w_cc = 0.5*(h_avg_w[k-1] + h_avg_w[k]);
152  uw_cc = 0.5*(h_avg_uw[k-1] + h_avg_uw[k]);
153  vw_cc = 0.5*(h_avg_vw[k-1] + h_avg_vw[k]);
154  ww_cc = 0.5*(h_avg_ww[k-1] + h_avg_ww[k]);
155  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
156  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
157  << h_avg_uu[k] - h_avg_u[k]*h_avg_u[k] << " " // u'u'
158  << h_avg_uv[k] - h_avg_u[k]*h_avg_v[k] << " " // u'v'
159  << h_avg_uw[k] - uface*h_avg_w[k] << " " // u'w'
160  << h_avg_vv[k] - h_avg_v[k]*h_avg_v[k] << " " // v'v'
161  << h_avg_vw[k] - vface*h_avg_w[k] << " " // v'w'
162  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " " // w'w'
163  << h_avg_uth[k] - h_avg_u[k]*h_avg_th[k] << " " // u'th'
164  << h_avg_vth[k] - h_avg_v[k]*h_avg_th[k] << " " // v'th'
165  << h_avg_wth[k] - h_avg_w[k]*thface << " " // w'th'
166  << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " " // th'th'
167  // Note: <u'_i u'_i u'_j> = <u_i u_i u_j>
168  // - <u_i u_i> * <u_j>
169  // - 2*<u_i> * <u_i u_j>
170  // + 2*<u_i>*<u_i> * <u_j>
171  << h_avg_uiuiu[k]
172  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_u[k]
173  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + w_cc*uw_cc)
174  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_u[k]
175  << " " // cell-centered (u'_i u'_i)u'
176  << h_avg_uiuiv[k]
177  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_v[k]
178  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + w_cc*vw_cc)
179  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_v[k]
180  << " " // cell-centered (u'_i u'_i)v'
181  << h_avg_uiuiw[k]
182  - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
183  - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
184  + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
185  << " " // face-centered (u'_i u'_i)w'
186  << h_avg_pu[k] - h_avg_p[k]*h_avg_u[k] << " " // cell-centered p'u'
187  << h_avg_pv[k] - h_avg_p[k]*h_avg_v[k] << " " // cell-centered p'v'
188  << h_avg_pw[k] - pface*h_avg_w[k] << " " // face-centered p'w'
189  << h_avg_wqv[k] - qvface*h_avg_w[k] << " "
190  << h_avg_wqc[k] - qcface*h_avg_w[k] << " "
191  << h_avg_wqr[k] - qrface*h_avg_w[k] << " "
192  << h_avg_wthv[k] - thvface*h_avg_w[k]
193  << std::endl;
194  } // loop over z
195 
196  // Write top face values, extrapolating scalar quantities
197  const int k = unstag_size;
198  Real uface = 1.5*h_avg_u[k-1] - 0.5*h_avg_u[k-2];
199  Real vface = 1.5*h_avg_v[k-1] - 0.5*h_avg_v[k-2];
200  Real thface = 1.5*h_avg_th[k-1] - 0.5*h_avg_th[k-2];
201  Real pface = 1.5*h_avg_p[k-1] - 0.5*h_avg_p[k-2];
202  Real qvface = 1.5*h_avg_qv[k-1] - 0.5*h_avg_qv[k-2];
203  Real qcface = 1.5*h_avg_qc[k-1] - 0.5*h_avg_qc[k-2];
204  Real qrface = 1.5*h_avg_qr[k-1] - 0.5*h_avg_qr[k-2];
205  Real uuface = 1.5*h_avg_uu[k-1] - 0.5*h_avg_uu[k-2];
206  Real vvface = 1.5*h_avg_vv[k-1] - 0.5*h_avg_vv[k-2];
207  Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
208  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
209  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
210  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
211  << 0 << " " // u'u'
212  << 0 << " " // u'v'
213  << h_avg_uw[k] - uface*h_avg_w[k] << " " // u'w'
214  << 0 << " " // v'v'
215  << h_avg_vw[k] - vface*h_avg_w[k] << " " // v'w'
216  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " " // w'w'
217  << 0 << " " // u'th'
218  << 0 << " " // v'th'
219  << h_avg_wth[k] - thface*h_avg_w[k] << " " // w'th'
220  << 0 << " " // th'th'
221  << 0 << " " // (u'_i u'_i)u'
222  << 0 << " " // (u'_i u'_i)v'
223  << h_avg_uiuiw[k]
224  - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
225  - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
226  + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
227  << " " // (u'_i u'_i)w'
228  << 0 << " " // pu'
229  << 0 << " " // pv'
230  << h_avg_pw[k] - pface*h_avg_w[k] << " " // pw'
231  << h_avg_wqv[k] - qvface*h_avg_w[k] << " "
232  << h_avg_wqc[k] - qcface*h_avg_w[k] << " "
233  << h_avg_wqr[k] - qrface*h_avg_w[k] << " "
234  << h_avg_wthv[k] - thvface*h_avg_w[k]
235  << std::endl;
236  } // if good
237  } // NumDataLogs
238 
239  if (NumDataLogs() > 3 && time > 0.) {
240  std::ostream& data_log3 = DataLog(3);
241  if (data_log3.good()) {
242  // Write the average stresses
243  for (int k = 0; k < unstag_size; k++) {
244  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
245  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
246  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
247  << h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
248  << h_avg_tau22[k] << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
249  << h_avg_sgshfx[k] << " "
250  << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
251  << h_avg_sgsdiss[k]
252  << std::endl;
253  } // loop over z
254  // Write top face values
255  Real NANval = 0.0;
256  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
257  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
258  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
259  << NANval << " " << NANval << " " << h_avg_tau13[unstag_size] << " "
260  << NANval << " " << h_avg_tau23[unstag_size] << " " << NANval << " "
261  << h_avg_sgshfx[unstag_size] << " "
262  << h_avg_sgsq1fx[unstag_size] << " " << h_avg_sgsq2fx[unstag_size] << " "
263  << NANval
264  << std::endl;
265  } // if good
266  } // if (NumDataLogs() > 3)
267  } // if IOProcessor
268  } // if (NumDataLogs() > 1)
269 }
void derive_diag_profiles_stag(amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
Definition: ERF_Write1DProfiles_stag.cpp:296
void derive_stress_profiles_stag(amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
Definition: ERF_Write1DProfiles_stag.cpp:605

◆ writeBuildInfo()

void ERF::writeBuildInfo ( std::ostream &  os)
static
138 {
139  std::string PrettyLine = std::string(78, '=') + "\n";
140  std::string OtherLine = std::string(78, '-') + "\n";
141  std::string SkipSpace = std::string(8, ' ');
142 
143  // build information
144  os << PrettyLine;
145  os << " ERF Build Information\n";
146  os << PrettyLine;
147 
148  os << "build date: " << buildInfoGetBuildDate() << "\n";
149  os << "build machine: " << buildInfoGetBuildMachine() << "\n";
150  os << "build dir: " << buildInfoGetBuildDir() << "\n";
151  os << "AMReX dir: " << buildInfoGetAMReXDir() << "\n";
152 
153  os << "\n";
154 
155  os << "COMP: " << buildInfoGetComp() << "\n";
156  os << "COMP version: " << buildInfoGetCompVersion() << "\n";
157 
158  os << "C++ compiler: " << buildInfoGetCXXName() << "\n";
159  os << "C++ flags: " << buildInfoGetCXXFlags() << "\n";
160 
161  os << "\n";
162 
163  os << "Link flags: " << buildInfoGetLinkFlags() << "\n";
164  os << "Libraries: " << buildInfoGetLibraries() << "\n";
165 
166  os << "\n";
167 
168  for (int n = 1; n <= buildInfoGetNumModules(); n++) {
169  os << buildInfoGetModuleName(n) << ": "
170  << buildInfoGetModuleVal(n) << "\n";
171  }
172 
173  os << "\n";
174  const char* githash1 = buildInfoGetGitHash(1);
175  const char* githash2 = buildInfoGetGitHash(2);
176  if (strlen(githash1) > 0) {
177  os << "ERF git hash: " << githash1 << "\n";
178  }
179  if (strlen(githash2) > 0) {
180  os << "AMReX git hash: " << githash2 << "\n";
181  }
182 
183  const char* buildgithash = buildInfoGetBuildGitHash();
184  const char* buildgitname = buildInfoGetBuildGitName();
185  if (strlen(buildgithash) > 0) {
186  os << buildgitname << " git hash: " << buildgithash << "\n";
187  }
188 
189  os << "\n";
190  os << " ERF Compile time variables: \n";
191 
192  os << "\n";
193  os << " ERF Defines: \n";
194 #ifdef _OPENMP
195  os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "ON"
196  << std::endl;
197 #else
198  os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "OFF"
199  << std::endl;
200 #endif
201 
202 #ifdef MPI_VERSION
203  os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
204  << MPI_VERSION << std::endl;
205 #else
206  os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
207  << "UNDEFINED" << std::endl;
208 #endif
209 
210 #ifdef MPI_SUBVERSION
211  os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
212  << MPI_SUBVERSION << std::endl;
213 #else
214  os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
215  << "UNDEFINED" << std::endl;
216 #endif
217 
218  os << "\n\n";
219 }

Referenced by main().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ WriteCheckpointFile()

void ERF::WriteCheckpointFile ( ) const

ERF function for writing a checkpoint file.

27 {
28  auto dCheckTime0 = amrex::second();
29 
30  // chk00010 write a checkpoint file with this root directory
31  // chk00010/Header this contains information you need to save (e.g., finest_level, t_new, etc.) and also
32  // the BoxArrays at each level
33  // chk00010/Level_0/
34  // chk00010/Level_1/
35  // etc. these subdirectories will hold the MultiFab data at each level of refinement
36 
37  // checkpoint file name, e.g., chk00010
38  const std::string& checkpointname = Concatenate(check_file,istep[0],5);
39 
40  Print() << "Writing native checkpoint " << checkpointname << "\n";
41 
42  const int nlevels = finest_level+1;
43 
44  // ---- prebuild a hierarchy of directories
45  // ---- dirName is built first. if dirName exists, it is renamed. then build
46  // ---- dirName/subDirPrefix_0 .. dirName/subDirPrefix_nlevels-1
47  // ---- if callBarrier is true, call ParallelDescriptor::Barrier()
48  // ---- after all directories are built
49  // ---- ParallelDescriptor::IOProcessor() creates the directories
50  PreBuildDirectorHierarchy(checkpointname, "Level_", nlevels, true);
51 
52  int ncomp_cons = vars_new[0][Vars::cons].nComp();
53 
54  // write Header file
55  if (ParallelDescriptor::IOProcessor()) {
56 
57  std::string HeaderFileName(checkpointname + "/Header");
58  VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
59  std::ofstream HeaderFile;
60  HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
61  HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out |
62  std::ofstream::trunc |
63  std::ofstream::binary);
64  if(! HeaderFile.good()) {
65  FileOpenFailed(HeaderFileName);
66  }
67 
68  HeaderFile.precision(17);
69 
70  // write out title line
71  HeaderFile << "Checkpoint file for ERF\n";
72 
73  // write out finest_level
74  HeaderFile << finest_level << "\n";
75 
76  // write the number of components
77  // for each variable we store
78 
79  // conservative, cell-centered vars
80  HeaderFile << ncomp_cons << "\n";
81 
82  // x-velocity on faces
83  HeaderFile << 1 << "\n";
84 
85  // y-velocity on faces
86  HeaderFile << 1 << "\n";
87 
88  // z-velocity on faces
89  HeaderFile << 1 << "\n";
90 
91  // write out array of istep
92  for (int i = 0; i < istep.size(); ++i) {
93  HeaderFile << istep[i] << " ";
94  }
95  HeaderFile << "\n";
96 
97  // write out array of dt
98  for (int i = 0; i < dt.size(); ++i) {
99  HeaderFile << dt[i] << " ";
100  }
101  HeaderFile << "\n";
102 
103  // write out array of t_new
104  for (int i = 0; i < t_new.size(); ++i) {
105  HeaderFile << t_new[i] << " ";
106  }
107  HeaderFile << "\n";
108 
109  // write the BoxArray at each level
110  for (int lev = 0; lev <= finest_level; ++lev) {
111  boxArray(lev).writeOn(HeaderFile);
112  HeaderFile << '\n';
113  }
114 
115  // Write separate file that tells how many components we have of the base state
116  std::string BaseStateFileName(checkpointname + "/num_base_state_comps");
117  std::ofstream BaseStateFile;
118  BaseStateFile.open(BaseStateFileName.c_str(), std::ofstream::out |
119  std::ofstream::trunc |
120  std::ofstream::binary);
121  if(! BaseStateFile.good()) {
122  FileOpenFailed(BaseStateFileName);
123  } else {
124  // write out number of components in base state
125  BaseStateFile << BaseState::num_comps << "\n";
126  BaseStateFile << base_state[0].nGrowVect() << "\n";
127  }
128  }
129 
130  // write the MultiFab data to, e.g., chk00010/Level_0/
131  // Here we make copies of the MultiFab with no ghost cells
132  for (int lev = 0; lev <= finest_level; ++lev)
133  {
134  MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
135  MultiFab::Copy(cons,vars_new[lev][Vars::cons],0,0,ncomp_cons,0);
136  VisMF::Write(cons, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));
137 
138  MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
139  MultiFab::Copy(xvel,vars_new[lev][Vars::xvel],0,0,1,0);
140  VisMF::Write(xvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));
141 
142  MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
143  MultiFab::Copy(yvel,vars_new[lev][Vars::yvel],0,0,1,0);
144  VisMF::Write(yvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));
145 
146  MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
147  MultiFab::Copy(zvel,vars_new[lev][Vars::zvel],0,0,1,0);
148  VisMF::Write(zvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));
149 
150  if (solverChoice.anelastic[lev] == 1) {
151  MultiFab ppinc(grids[lev],dmap[lev],1,0);
152  MultiFab::Copy(ppinc,pp_inc[lev],0,0,1,0);
153  VisMF::Write(ppinc, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PP_Inc"));
154 
155  MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
156  MultiFab::Copy(gpx,gradp[lev][GpVars::gpx],0,0,1,0);
157  VisMF::Write(gpx, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpx"));
158 
159  MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
160  MultiFab::Copy(gpy,gradp[lev][GpVars::gpy],0,0,1,0);
161  VisMF::Write(gpy, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpy"));
162 
163  MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
164  MultiFab::Copy(gpz,gradp[lev][GpVars::gpz],0,0,1,0);
165  VisMF::Write(gpz, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpz"));
166  }
167 
168  // Note that we write the ghost cells of the base state (unlike above)
169  IntVect ng_base = base_state[lev].nGrowVect();
170  int ncomp_base = base_state[lev].nComp();
171  MultiFab base(grids[lev],dmap[lev],ncomp_base,ng_base);
172  MultiFab::Copy(base,base_state[lev],0,0,ncomp_base,ng_base);
173  VisMF::Write(base, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "BaseState"));
174 
175  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
176  // Note that we also write the ghost cells of z_phys_nd
177  IntVect ng = z_phys_nd[lev]->nGrowVect();
178  MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
179  MultiFab::Copy(z_height,*z_phys_nd[lev],0,0,1,ng);
180  VisMF::Write(z_height, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z_Phys_nd"));
181  }
182 
183  // We must read and write qmoist with ghost cells because we don't directly impose BCs on these vars
184  // Write the moisture model restart variables
185  std::vector<int> qmoist_indices;
186  std::vector<std::string> qmoist_names;
187  micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
188  int qmoist_nvar = qmoist_indices.size();
189  for (int var = 0; var < qmoist_nvar; var++) {
190  const int ncomp = 1;
191  IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
192  MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
193  MultiFab::Copy(moist_vars,*(qmoist[lev][qmoist_indices[var]]),0,0,ncomp,ng_moist);
194  VisMF::Write(moist_vars, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", qmoist_names[var]));
195  }
196 
197 #if defined(ERF_USE_WINDFARM)
198  if(solverChoice.windfarm_type == WindFarmType::Fitch or
199  solverChoice.windfarm_type == WindFarmType::EWP or
200  solverChoice.windfarm_type == WindFarmType::SimpleAD){
201  IntVect ng_turb = Nturb[lev].nGrowVect();
202  MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng_turb);
203  MultiFab::Copy(mf_Nturb,Nturb[lev],0,0,1,ng_turb);
204  VisMF::Write(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "NumTurb"));
205  }
206 #endif
207 
208  if (solverChoice.lsm_type != LandSurfaceType::None) {
209  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
210  BoxArray ba = lsm_data[lev][mvar]->boxArray();
211  DistributionMapping dm = lsm_data[lev][mvar]->DistributionMap();
212  IntVect ng = lsm_data[lev][mvar]->nGrowVect();
213  int nvar = lsm_data[lev][mvar]->nComp();
214  MultiFab lsm_vars(ba,dm,nvar,ng);
215  MultiFab::Copy(lsm_vars,*(lsm_data[lev][mvar]),0,0,nvar,ng);
216  VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmVars"));
217  }
218  }
219 
220  IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
221  MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
222  MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_x],0,0,1,ng);
223  VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_mx"));
224 
225 #if 0
227  MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_y],0,0,1,ng);
228  VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_my"));
229  }
230 #endif
231 
232  ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
233  MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
234  MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_x],0,0,1,ng);
235  VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_ux"));
236 
237 #if 0
239  MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_y],0,0,1,ng);
240  VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_uy"));
241  }
242 #endif
243 
244  ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
245  MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
246  MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_x],0,0,1,ng);
247  VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vx"));
248 
249 #if 0
251  MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_y],0,0,1,ng);
252  VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vy"));
253  }
254 #endif
255 
256  if (m_SurfaceLayer) {
257  amrex::Print() << "Writing SurfaceLayer variables at level " << lev << std::endl;
258  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
259  MultiFab m_var(ba2d[lev],dmap[lev],1,ng);
260  MultiFab* src = nullptr;
261 
262  // U*
263  src = m_SurfaceLayer->get_u_star(lev);
264  MultiFab::Copy(m_var,*src,0,0,1,ng);
265  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Ustar"));
266 
267  // W*
268  src = m_SurfaceLayer->get_w_star(lev);
269  MultiFab::Copy(m_var,*src,0,0,1,ng);
270  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Wstar"));
271 
272  // T*
273  src = m_SurfaceLayer->get_t_star(lev);
274  MultiFab::Copy(m_var,*src,0,0,1,ng);
275  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Tstar"));
276 
277  // Q*
278  src = m_SurfaceLayer->get_q_star(lev);
279  MultiFab::Copy(m_var,*src,0,0,1,ng);
280  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qstar"));
281 
282  // Olen
283  src = m_SurfaceLayer->get_olen(lev);
284  MultiFab::Copy(m_var,*src,0,0,1,ng);
285  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Olen"));
286 
287  // Qsurf
288  src = m_SurfaceLayer->get_q_surf(lev);
289  MultiFab::Copy(m_var,*src,0,0,1,ng);
290  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qsurf"));
291 
292  // PBLH
293  src = m_SurfaceLayer->get_pblh(lev);
294  MultiFab::Copy(m_var,*src,0,0,1,ng);
295  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PBLH"));
296 
297  // Z0
298  src = m_SurfaceLayer->get_z0(lev);
299  MultiFab::Copy(m_var,*src,0,0,1,ng);
300  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z0"));
301  }
302 
303  if (sst_lev[lev][0]) {
304  int ntimes = 1;
305  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
306  MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
307  for (int nt(0); nt<ntimes; ++nt) {
308  MultiFab::Copy(sst_at_t,*sst_lev[lev][nt],0,0,1,ng);
309  VisMF::Write(sst_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
310  "SST_" + std::to_string(nt)));
311  }
312  }
313 
314  if (tsk_lev[lev][0]) {
315  int ntimes = 1;
316  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
317  MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
318  for (int nt(0); nt<ntimes; ++nt) {
319  MultiFab::Copy(tsk_at_t,*tsk_lev[lev][nt],0,0,1,ng);
320  VisMF::Write(tsk_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
321  "TSK_" + std::to_string(nt)));
322  }
323  }
324 
325  {
326  int ntimes = 1;
327  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
328  MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
329  for (int nt(0); nt<ntimes; ++nt) {
330  for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
331  const Box& bx = mfi.growntilebox();
332  Array4<int> const& src_arr = lmask_lev[lev][nt]->array(mfi);
333  Array4<Real> const& dst_arr = lmask_at_t.array(mfi);
334  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
335  {
336  dst_arr(i,j,k) = Real(src_arr(i,j,k));
337  });
338  }
339  VisMF::Write(lmask_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
340  "LMASK_" + std::to_string(nt)));
341  }
342  }
343 
344  IntVect ngv = ng; ngv[2] = 0;
345 
346  // Write lat/lon if it exists
347  if (lat_m[lev] && lon_m[lev] && solverChoice.has_lat_lon) {
348  amrex::Print() << "Writing Lat/Lon variables at level " << lev << std::endl;
349  MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
350  MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
351  MultiFab::Copy(lat,*lat_m[lev],0,0,1,ngv);
352  MultiFab::Copy(lon,*lon_m[lev],0,0,1,ngv);
353  VisMF::Write(lat, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LAT"));
354  VisMF::Write(lon, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LON"));
355  }
356 
357 
358 #ifdef ERF_USE_NETCDF
359  // Write sinPhi and cosPhi if it exists
360  if (cosPhi_m[lev] && sinPhi_m[lev] && solverChoice.variable_coriolis) {
361  amrex::Print() << "Writing Coriolis factors at level " << lev << std::endl;
362  MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
363  MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
364  MultiFab::Copy(sphi,*sinPhi_m[lev],0,0,1,ngv);
365  MultiFab::Copy(cphi,*cosPhi_m[lev],0,0,1,ngv);
366  VisMF::Write(sphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "SinPhi"));
367  VisMF::Write(cphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "CosPhi"));
368  }
369 
370  if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
371  amrex::Print() << "Writing C1H/C2H/MUB variables at level " << lev << std::endl;
372  MultiFab tmp1d(ba1d[0],dmap[0],1,0);
373 
374  MultiFab::Copy(tmp1d,*mf_C1H,0,0,1,0);
375  VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C1H"));
376 
377  MultiFab::Copy(tmp1d,*mf_C2H,0,0,1,0);
378  VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C2H"));
379 
380  MultiFab tmp2d(ba2d[0],dmap[0],1,mf_MUB->nGrowVect());
381 
382  MultiFab::Copy(tmp2d,*mf_MUB,0,0,1,mf_MUB->nGrowVect());
383  VisMF::Write(tmp2d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MUB"));
384  }
385 #endif
386 
387  } // for lev
388 
389 #ifdef ERF_USE_PARTICLES
390  particleData.Checkpoint(checkpointname);
391 #endif
392 
393 #if 0
394 #ifdef ERF_USE_NETCDF
395  // Write bdy_data files
396  if ( ParallelDescriptor::IOProcessor() &&
397  ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
399  {
400  // Vector dimensions
401  int num_time = bdy_data_xlo.size();
402  int num_var = bdy_data_xlo[0].size();
403 
404  // Open header file and write to it
405  std::ofstream bdy_h_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_H"));
406  bdy_h_file << std::setprecision(1) << std::fixed;
407  bdy_h_file << num_time << "\n";
408  bdy_h_file << num_var << "\n";
409  bdy_h_file << start_bdy_time << "\n";
410  bdy_h_file << bdy_time_interval << "\n";
411  bdy_h_file << real_width << "\n";
412  for (int ivar(0); ivar<num_var; ++ivar) {
413  bdy_h_file << bdy_data_xlo[0][ivar].box() << "\n";
414  bdy_h_file << bdy_data_xhi[0][ivar].box() << "\n";
415  bdy_h_file << bdy_data_ylo[0][ivar].box() << "\n";
416  bdy_h_file << bdy_data_yhi[0][ivar].box() << "\n";
417  }
418 
419  // Open data file and write to it
420  std::ofstream bdy_d_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_D"));
421  for (int itime(0); itime<num_time; ++itime) {
422  if (bdy_data_xlo[itime].size() > 0) {
423  for (int ivar(0); ivar<num_var; ++ivar) {
424  bdy_data_xlo[itime][ivar].writeOn(bdy_d_file,0,1);
425  bdy_data_xhi[itime][ivar].writeOn(bdy_d_file,0,1);
426  bdy_data_ylo[itime][ivar].writeOn(bdy_d_file,0,1);
427  bdy_data_yhi[itime][ivar].writeOn(bdy_d_file,0,1);
428  }
429  }
430  }
431  }
432 #endif
433 #endif
434 
435  if (verbose > 0)
436  {
437  auto dCheckTime = amrex::second() - dCheckTime0;
438  ParallelDescriptor::ReduceRealMax(dCheckTime,ParallelDescriptor::IOProcessorNumber());
439  amrex::Print() << "Checkpoint write time = " << dCheckTime << " seconds." << '\n';
440  }
441 }

◆ WriteGenericPlotfileHeaderWithTerrain()

void ERF::WriteGenericPlotfileHeaderWithTerrain ( std::ostream &  HeaderFile,
int  nlevels,
const amrex::Vector< amrex::BoxArray > &  bArray,
const amrex::Vector< std::string > &  varnames,
const amrex::Vector< amrex::Geometry > &  my_geom,
amrex::Real  time,
const amrex::Vector< int > &  level_steps,
const amrex::Vector< amrex::IntVect > &  my_ref_ratio,
const std::string &  versionName,
const std::string &  levelPrefix,
const std::string &  mfPrefix 
) const
1841 {
1842  AMREX_ALWAYS_ASSERT(nlevels <= bArray.size());
1843  AMREX_ALWAYS_ASSERT(nlevels <= my_ref_ratio.size()+1);
1844  AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
1845 
1846  HeaderFile.precision(17);
1847 
1848  // ---- this is the generic plot file type name
1849  HeaderFile << versionName << '\n';
1850 
1851  HeaderFile << varnames.size() << '\n';
1852 
1853  for (int ivar = 0; ivar < varnames.size(); ++ivar) {
1854  HeaderFile << varnames[ivar] << "\n";
1855  }
1856  HeaderFile << AMREX_SPACEDIM << '\n';
1857  HeaderFile << my_time << '\n';
1858  HeaderFile << finest_level << '\n';
1859  for (int i = 0; i < AMREX_SPACEDIM; ++i) {
1860  HeaderFile << my_geom[0].ProbLo(i) << ' ';
1861  }
1862  HeaderFile << '\n';
1863  for (int i = 0; i < AMREX_SPACEDIM; ++i) {
1864  HeaderFile << my_geom[0].ProbHi(i) << ' ';
1865  }
1866  HeaderFile << '\n';
1867  for (int i = 0; i < finest_level; ++i) {
1868  HeaderFile << my_ref_ratio[i][0] << ' ';
1869  }
1870  HeaderFile << '\n';
1871  for (int i = 0; i <= finest_level; ++i) {
1872  HeaderFile << my_geom[i].Domain() << ' ';
1873  }
1874  HeaderFile << '\n';
1875  for (int i = 0; i <= finest_level; ++i) {
1876  HeaderFile << level_steps[i] << ' ';
1877  }
1878  HeaderFile << '\n';
1879  for (int i = 0; i <= finest_level; ++i) {
1880  for (int k = 0; k < AMREX_SPACEDIM; ++k) {
1881  HeaderFile << my_geom[i].CellSize()[k] << ' ';
1882  }
1883  HeaderFile << '\n';
1884  }
1885  HeaderFile << (int) my_geom[0].Coord() << '\n';
1886  HeaderFile << "0\n";
1887 
1888  for (int level = 0; level <= finest_level; ++level) {
1889  HeaderFile << level << ' ' << bArray[level].size() << ' ' << my_time << '\n';
1890  HeaderFile << level_steps[level] << '\n';
1891 
1892  const IntVect& domain_lo = my_geom[level].Domain().smallEnd();
1893  for (int i = 0; i < bArray[level].size(); ++i)
1894  {
1895  // Need to shift because the RealBox ctor we call takes the
1896  // physical location of index (0,0,0). This does not affect
1897  // the usual cases where the domain index starts with 0.
1898  const Box& b = shift(bArray[level][i], -domain_lo);
1899  RealBox loc = RealBox(b, my_geom[level].CellSize(), my_geom[level].ProbLo());
1900  for (int n = 0; n < AMREX_SPACEDIM; ++n) {
1901  HeaderFile << loc.lo(n) << ' ' << loc.hi(n) << '\n';
1902  }
1903  }
1904 
1905  HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n';
1906  }
1907  HeaderFile << "1" << "\n";
1908  HeaderFile << "3" << "\n";
1909  HeaderFile << "amrexvec_nu_x" << "\n";
1910  HeaderFile << "amrexvec_nu_y" << "\n";
1911  HeaderFile << "amrexvec_nu_z" << "\n";
1912  std::string mf_nodal_prefix = "Nu_nd";
1913  for (int level = 0; level <= finest_level; ++level) {
1914  HeaderFile << MultiFabHeaderPath(level, levelPrefix, mf_nodal_prefix) << '\n';
1915  }
1916 }
Coord
Definition: ERF_DataStruct.H:86

◆ writeJobInfo()

void ERF::writeJobInfo ( const std::string &  dir) const
10 {
11  // job_info file with details about the run
12  std::ofstream jobInfoFile;
13  std::string FullPathJobInfoFile = dir;
14  FullPathJobInfoFile += "/job_info";
15  jobInfoFile.open(FullPathJobInfoFile.c_str(), std::ios::out);
16 
17  std::string PrettyLine = "==================================================="
18  "============================\n";
19  std::string OtherLine = "----------------------------------------------------"
20  "----------------------------\n";
21  std::string SkipSpace = " ";
22 
23  // job information
24  jobInfoFile << PrettyLine;
25  jobInfoFile << " ERF Job Information\n";
26  jobInfoFile << PrettyLine;
27 
28  jobInfoFile << "inputs file: " << inputs_name << "\n\n";
29 
30  jobInfoFile << "number of MPI processes: "
31  << ParallelDescriptor::NProcs() << "\n";
32 #ifdef _OPENMP
33  jobInfoFile << "number of threads: " << omp_get_max_threads() << "\n";
34 #endif
35 
36  jobInfoFile << "\n";
37  jobInfoFile << "CPU time used since start of simulation (CPU-hours): "
38  << getCPUTime() / 3600.0;
39 
40  jobInfoFile << "\n\n";
41 
42  // plotfile information
43  jobInfoFile << PrettyLine;
44  jobInfoFile << " Plotfile Information\n";
45  jobInfoFile << PrettyLine;
46 
47  time_t now = time(nullptr);
48 
49  // Convert now to tm struct for local timezone
50  tm* localtm = localtime(&now);
51  jobInfoFile << "output data / time: " << asctime(localtm);
52 
53  std::string currentDir = FileSystem::CurrentPath();
54  jobInfoFile << "output dir: " << currentDir << "\n";
55 
56  jobInfoFile << "\n\n";
57 
58  // build information
59  jobInfoFile << PrettyLine;
60  jobInfoFile << " Build Information\n";
61  jobInfoFile << PrettyLine;
62 
63  jobInfoFile << "build date: " << buildInfoGetBuildDate() << "\n";
64  jobInfoFile << "build machine: " << buildInfoGetBuildMachine() << "\n";
65  jobInfoFile << "build dir: " << buildInfoGetBuildDir() << "\n";
66  jobInfoFile << "AMReX dir: " << buildInfoGetAMReXDir() << "\n";
67 
68  jobInfoFile << "\n";
69 
70  jobInfoFile << "COMP: " << buildInfoGetComp() << "\n";
71  jobInfoFile << "COMP version: " << buildInfoGetCompVersion() << "\n";
72 
73  jobInfoFile << "\n";
74 
75  for (int n = 1; n <= buildInfoGetNumModules(); n++) {
76  jobInfoFile << buildInfoGetModuleName(n) << ": "
77  << buildInfoGetModuleVal(n) << "\n";
78  }
79 
80  jobInfoFile << "\n";
81 
82  const char* githash1 = buildInfoGetGitHash(1);
83  const char* githash2 = buildInfoGetGitHash(2);
84  if (strlen(githash1) > 0) {
85  jobInfoFile << "ERF git hash: " << githash1 << "\n";
86  }
87  if (strlen(githash2) > 0) {
88  jobInfoFile << "AMReX git hash: " << githash2 << "\n";
89  }
90 
91  const char* buildgithash = buildInfoGetBuildGitHash();
92  const char* buildgitname = buildInfoGetBuildGitName();
93  if (strlen(buildgithash) > 0) {
94  jobInfoFile << buildgitname << " git hash: " << buildgithash << "\n";
95  }
96 
97  jobInfoFile << "\n\n";
98 
99  // grid information
100  jobInfoFile << PrettyLine;
101  jobInfoFile << " Grid Information\n";
102  jobInfoFile << PrettyLine;
103 
104  int f_lev = finest_level;
105 
106  for (int i = 0; i <= f_lev; i++) {
107  jobInfoFile << " level: " << i << "\n";
108  jobInfoFile << " number of boxes = " << grids[i].size() << "\n";
109  jobInfoFile << " maximum zones = ";
110  for (int n = 0; n < AMREX_SPACEDIM; n++) {
111  jobInfoFile << geom[i].Domain().length(n) << " ";
112  }
113  jobInfoFile << "\n\n";
114  }
115 
116  jobInfoFile << " Boundary conditions\n";
117 
118  jobInfoFile << " -x: " << domain_bc_type[0] << "\n";
119  jobInfoFile << " +x: " << domain_bc_type[3] << "\n";
120  jobInfoFile << " -y: " << domain_bc_type[1] << "\n";
121  jobInfoFile << " +y: " << domain_bc_type[4] << "\n";
122  jobInfoFile << " -z: " << domain_bc_type[2] << "\n";
123  jobInfoFile << " +z: " << domain_bc_type[5] << "\n";
124 
125  jobInfoFile << "\n\n";
126 
127  // runtime parameters
128  jobInfoFile << PrettyLine;
129  jobInfoFile << " Inputs File Parameters\n";
130  jobInfoFile << PrettyLine;
131 
132  ParmParse::dumpTable(jobInfoFile, true);
133  jobInfoFile.close();
134 }
std::string inputs_name
Definition: main.cpp:14
static amrex::Real getCPUTime()
Definition: ERF.H:1499
Here is the call graph for this function:

◆ WriteLinePlot()

void ERF::WriteLinePlot ( const std::string &  filename,
amrex::Vector< std::array< amrex::Real, 2 >> &  points_xy 
)
718 {
719  std::ofstream ofs(filename);
720  if (!ofs.is_open()) {
721  amrex::Print() << "Error: Could not open file " << filename << " for writing.\n";
722  return;
723  }
724 
725  ofs << std::setprecision(10) << std::scientific;
726  ofs << "# x y\n";
727 
728  for (const auto& p : points_xy) {
729  ofs << p[0] << " " << p[1] << "\n";
730  }
731 
732  ofs.close();
733 
734  amrex::Print() << "Line plot data written to " << filename << "\n";
735 }

◆ WriteMultiLevelPlotfileWithTerrain()

void ERF::WriteMultiLevelPlotfileWithTerrain ( const std::string &  plotfilename,
int  nlevels,
const amrex::Vector< const amrex::MultiFab * > &  mf,
const amrex::Vector< const amrex::MultiFab * > &  mf_nd,
const amrex::Vector< std::string > &  varnames,
const amrex::Vector< amrex::Geometry > &  my_geom,
amrex::Real  time,
const amrex::Vector< int > &  level_steps,
const amrex::Vector< amrex::IntVect > &  my_ref_ratio,
const std::string &  versionName = "HyperCLaw-V1.1",
const std::string &  levelPrefix = "Level_",
const std::string &  mfPrefix = "Cell",
const amrex::Vector< std::string > &  extra_dirs = amrex::Vector<std::string>() 
) const
1755 {
1756  BL_PROFILE("WriteMultiLevelPlotfileWithTerrain()");
1757 
1758  AMREX_ALWAYS_ASSERT(nlevels <= mf.size());
1759  AMREX_ALWAYS_ASSERT(nlevels <= rr.size()+1);
1760  AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
1761  AMREX_ALWAYS_ASSERT(mf[0]->nComp() == varnames.size());
1762 
1763  bool callBarrier(false);
1764  PreBuildDirectorHierarchy(plotfilename, levelPrefix, nlevels, callBarrier);
1765  if (!extra_dirs.empty()) {
1766  for (const auto& d : extra_dirs) {
1767  const std::string ed = plotfilename+"/"+d;
1768  PreBuildDirectorHierarchy(ed, levelPrefix, nlevels, callBarrier);
1769  }
1770  }
1771  ParallelDescriptor::Barrier();
1772 
1773  if (ParallelDescriptor::MyProc() == ParallelDescriptor::NProcs()-1) {
1774  Vector<BoxArray> boxArrays(nlevels);
1775  for(int level(0); level < boxArrays.size(); ++level) {
1776  boxArrays[level] = mf[level]->boxArray();
1777  }
1778 
1779  auto f = [=]() {
1780  VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
1781  std::string HeaderFileName(plotfilename + "/Header");
1782  std::ofstream HeaderFile;
1783  HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
1784  HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out |
1785  std::ofstream::trunc |
1786  std::ofstream::binary);
1787  if( ! HeaderFile.good()) FileOpenFailed(HeaderFileName);
1788  WriteGenericPlotfileHeaderWithTerrain(HeaderFile, nlevels, boxArrays, varnames,
1789  my_geom, time, level_steps, rr, versionName,
1790  levelPrefix, mfPrefix);
1791  };
1792 
1793  if (AsyncOut::UseAsyncOut()) {
1794  AsyncOut::Submit(std::move(f));
1795  } else {
1796  f();
1797  }
1798  }
1799 
1800  std::string mf_nodal_prefix = "Nu_nd";
1801  for (int level = 0; level <= finest_level; ++level)
1802  {
1803  if (AsyncOut::UseAsyncOut()) {
1804  VisMF::AsyncWrite(*mf[level],
1805  MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix),
1806  true);
1807  VisMF::AsyncWrite(*mf_nd[level],
1808  MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix),
1809  true);
1810  } else {
1811  const MultiFab* data;
1812  std::unique_ptr<MultiFab> mf_tmp;
1813  if (mf[level]->nGrowVect() != 0) {
1814  mf_tmp = std::make_unique<MultiFab>(mf[level]->boxArray(),
1815  mf[level]->DistributionMap(),
1816  mf[level]->nComp(), 0, MFInfo(),
1817  mf[level]->Factory());
1818  MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0);
1819  data = mf_tmp.get();
1820  } else {
1821  data = mf[level];
1822  }
1823  VisMF::Write(*data , MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix));
1824  VisMF::Write(*mf_nd[level], MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix));
1825  }
1826  }
1827 }
void WriteGenericPlotfileHeaderWithTerrain(std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
Definition: ERF_Plotfile.cpp:1830

◆ WriteMyEBSurface()

void ERF::WriteMyEBSurface ( )
6 {
7  using namespace amrex;
8 
9  amrex::Print() << "Writing the geometry to a vtp file.\n" << std::endl;
10 
11  // Only write at the finest level!
12  int lev = finest_level;
13 
14  BoxArray & ba = grids[lev];
15  DistributionMapping & dm = dmap[lev];
16 
17  const EBFArrayBoxFactory* ebfact = &EBFactory(lev);
18 
19  WriteEBSurface(ba,dm,Geom(lev),ebfact);
20 }
Definition: ERF_ConsoleIO.cpp:12
Here is the call graph for this function:

◆ writeNow()

bool ERF::writeNow ( const amrex::Real  cur_time,
const int  nstep,
const int  plot_int,
const amrex::Real  plot_per,
const amrex::Real  dt_0,
amrex::Real last_file_time 
)
2768 {
2769  bool write_now = false;
2770 
2771  if ( plot_int > 0) {
2772 
2773  write_now = (nstep % plot_int == 0);
2774 
2775  } else if (plot_per > 0.0) {
2776 
2777  amrex::Print() << "CUR NEXT PER " << cur_time << " " << next_file_time << " " << plot_per << std::endl;
2778 
2779  // Only write now if nstep newly matches the number of elapsed periods
2780  write_now = (cur_time > (next_file_time - Real(0.1)*dt_0));
2781  }
2782 
2783  return write_now;
2784 }

◆ WriteSubvolume()

void ERF::WriteSubvolume ( amrex::Vector< std::string >  subvol_var_names)
145 {
146  ParmParse pp("erf.subvol");
147 
148  Vector<Real> origin;
149  Vector< int> ncell;
150  Vector<Real> delta;
151 
152  // **************************************************************
153  // Read in the origin, number of cells in each dir, and resolution
154  // **************************************************************
155 
156  int n1 = pp.countval("origin");
157  int n2 = pp.countval("nxnynz");
158  int n3 = pp.countval("dxdydz");
159 
160  if (n1 != n2 || n1 != n3 || n2 != n3) {
161  amrex::Abort("WriteSubvolume: must have same number of entries in origin, nxnynz, and dxdydz.");
162  }
163  if ( n1%AMREX_SPACEDIM != 0) {
164  amrex::Abort("WriteSubvolume: origin, nxnynz, and dxdydz must have multiples of AMReX_SPACEDIM");
165  }
166  int nsub = n1/AMREX_SPACEDIM;
167 
168  if (nsub == 1) {
169  amrex::Print() << "WriteSubvolume:There is " << nsub << " subvolume specified" << std::endl;
170  } else {
171  amrex::Print() << "WriteSubvolume:There are " << nsub << " subvolumes specified" << std::endl;
172  }
173  for (int isub = 0; isub < nsub; isub++) {
174 
175  int lev_for_sub = 0;
176  int offset = isub * AMREX_SPACEDIM;
177 
178  pp.getarr("origin",origin,offset,AMREX_SPACEDIM);
179  pp.getarr("nxnynz", ncell,offset,AMREX_SPACEDIM);
180  pp.getarr("dxdydz", delta,offset,AMREX_SPACEDIM);
181 
182  bool found = false;
183  for (int i = 0; i <= finest_level; i++) {
184  if (!found) {
185  if (almostEqual(delta[offset+0],geom[i].CellSize(0)) &&
186  almostEqual(delta[offset+1],geom[i].CellSize(1)) &&
187  almostEqual(delta[offset+2],geom[i].CellSize(2)) ) {
188 
189  amrex::Print() << "WriteSubvolume:Resolution specified matches that of level " << i << std::endl;
190  found = true;
191  lev_for_sub = i;
192  }
193  }
194  }
195 
196  if (!found) {
197  amrex::Abort("Resolution specified for subvol does not match the resolution of any of the levels.");
198  }
199 
200  // **************************************************************
201  // Now that we know which level we're at, we can figure out which (i,j,k) the origin corresponds to
202  // Note we use 1.0001 as a fudge factor since the division of two reals --> integer will do a floor
203  // **************************************************************
204  int i0 = static_cast<int>((origin[offset+0] - geom[lev_for_sub].ProbLo(0)) * 1.0001 / delta[offset+0]);
205  int j0 = static_cast<int>((origin[offset+1] - geom[lev_for_sub].ProbLo(1)) * 1.0001 / delta[offset+1]);
206  int k0 = static_cast<int>((origin[offset+2] - geom[lev_for_sub].ProbLo(2)) * 1.0001 / delta[offset+2]);
207 
208  found = false;
209  if (almostEqual(geom[lev_for_sub].ProbLo(0)+i0*delta[offset+0],origin[offset+0]) &&
210  almostEqual(geom[lev_for_sub].ProbLo(1)+j0*delta[offset+1],origin[offset+1]) &&
211  almostEqual(geom[lev_for_sub].ProbLo(2)+k0*delta[offset+2],origin[offset+2]) )
212  {
213  amrex::Print() << "WriteSubvolume:Specified origin is the lower left corner of cell " << IntVect(i0,j0,k0) << std::endl;
214  found = true;
215  }
216 
217  if (!found) {
218  amrex::Abort("Origin specified does not correspond to a node at this level.");
219  }
220 
221  Box domain(geom[lev_for_sub].Domain());
222 
223  Box bx(IntVect(i0,j0,k0),IntVect(i0+ncell[offset+0]-1,j0+ncell[offset+1]-1,k0+ncell[offset+2]-1));
224  amrex::Print() << "WriteSubvolume:Box requested is " << bx << std::endl;
225 
226  if (!domain.contains(bx))
227  {
228  amrex::Abort("WriteSubvolume:Box requested is larger than the existing domain");
229  }
230 
231  Vector<int> cs(AMREX_SPACEDIM);
232  int count = pp.countval("chunk_size");
233  if (count > 0) {
234  pp.queryarr("chunk_size",cs,0,AMREX_SPACEDIM);
235  } else {
236  cs[0] = max_grid_size[0][0];
237  cs[1] = max_grid_size[0][1];
238  cs[2] = max_grid_size[0][2];
239  }
240  IntVect chunk_size(cs[0],cs[1],cs[2]);
241 
242  BoxArray ba(bx);
243  ba.maxSize(chunk_size);
244 
245  amrex::Print() << "WriteSubvolume:BoxArray is " << ba << std::endl;
246 
247  Vector<std::string> varnames;
248  varnames.insert(varnames.end(), subvol_var_names.begin(), subvol_var_names.end());
249 
250  int ncomp_mf = subvol_var_names.size();
251 
252  DistributionMapping dm(ba);
253 
254  MultiFab mf(ba, dm, ncomp_mf, 0);
255 
256  int mf_comp = 0;
257 
258  // *****************************************************************************************
259 
260  // First, copy any of the conserved state variables into the output plotfile
261  for (int i = 0; i < cons_names.size(); ++i) {
262  if (containerHasElement(subvol_var_names, cons_names[i])) {
263  mf.ParallelCopy(vars_new[lev_for_sub][Vars::cons],i,mf_comp,1,1,0);
264  mf_comp++;
265  }
266  }
267 
268  // *****************************************************************************************
269 
270  if (containerHasElement(subvol_var_names, "x_velocity") ||
271  containerHasElement(subvol_var_names, "y_velocity") ||
272  containerHasElement(subvol_var_names, "z_velocity"))
273  {
274  MultiFab mf_cc_vel(grids[lev_for_sub], dmap[lev_for_sub], AMREX_SPACEDIM, 0);
275  average_face_to_cellcenter(mf_cc_vel,0,
276  Array<const MultiFab*,3>{&vars_new[lev_for_sub][Vars::xvel],
277  &vars_new[lev_for_sub][Vars::yvel],
278  &vars_new[lev_for_sub][Vars::zvel]});
279  if (containerHasElement(subvol_var_names, "x_velocity")) {
280  mf.ParallelCopy(mf_cc_vel,0,mf_comp,1,0,0);
281  mf_comp++;
282  }
283  if (containerHasElement(subvol_var_names, "y_velocity")) {
284  mf.ParallelCopy(mf_cc_vel,1,mf_comp,1,0,0);
285  mf_comp++;
286  }
287  if (containerHasElement(subvol_var_names, "z_velocity")) {
288  mf.ParallelCopy(mf_cc_vel,2,mf_comp,1,0,0);
289  mf_comp++;
290  }
291  }
292 
293  // *****************************************************************************************
294 
295  // Finally, check for any derived quantities and compute them, inserting
296  // them into our output multifab
297  auto calculate_derived = [&](const std::string& der_name,
298  MultiFab& src_mf,
299  decltype(derived::erf_dernull)& der_function)
300  {
301  if (containerHasElement(subvol_var_names, der_name)) {
302  MultiFab dmf(src_mf.boxArray(), src_mf.DistributionMap(), 1, 0);
303 #ifdef _OPENMP
304 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
305 #endif
306  for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
307  {
308  const Box& tbx = mfi.tilebox();
309  auto& dfab = dmf[mfi];
310  auto& sfab = src_mf[mfi];
311  der_function(tbx, dfab, 0, 1, sfab, Geom(lev_for_sub), t_new[0], nullptr, lev_for_sub);
312  }
313  mf.ParallelCopy(dmf,0,mf_comp,1,0,0);
314  mf_comp++;
315  }
316  };
317 
318  // *****************************************************************************************
319  // NOTE: All derived variables computed below **MUST MATCH THE ORDER** of "derived_names"
320  // defined in ERF.H
321  // *****************************************************************************************
322 
323  calculate_derived("soundspeed", vars_new[lev_for_sub][Vars::cons], derived::erf_dersoundspeed);
324  if (solverChoice.moisture_type != MoistureType::None) {
325  calculate_derived("temp", vars_new[lev_for_sub][Vars::cons], derived::erf_dermoisttemp);
326  } else {
327  calculate_derived("temp", vars_new[lev_for_sub][Vars::cons], derived::erf_dertemp);
328  }
329  calculate_derived("theta", vars_new[lev_for_sub][Vars::cons], derived::erf_dertheta);
330  calculate_derived("KE", vars_new[lev_for_sub][Vars::cons], derived::erf_derKE);
331  calculate_derived("scalar", vars_new[lev_for_sub][Vars::cons], derived::erf_derscalar);
332 
333  // *****************************************************************************************
334 
335  std::string subvol_filename = Concatenate(subvol_file + "_" + std::to_string(isub) + "_", istep[0], 5);
336 
337  Real time = t_new[lev_for_sub];
338 
339  amrex::Print() <<"Writing subvolume into " << subvol_filename << std::endl;
340  WriteSingleLevelPlotfile(subvol_filename,mf,varnames,geom[lev_for_sub],time,istep[0]);
341 
342 
343  } // isub
344 }
real(c_double), private cs
Definition: ERF_module_mp_morr_two_moment.F90:203
Here is the call graph for this function:

◆ WriteVTKPolyline()

void ERF::WriteVTKPolyline ( const std::string &  filename,
amrex::Vector< std::array< amrex::Real, 2 >> &  points_xy 
)
671 {
672  std::ofstream vtkfile(filename);
673  if (!vtkfile.is_open()) {
674  std::cerr << "Error: Cannot open file " << filename << std::endl;
675  return;
676  }
677 
678  int num_points = points_xy.size();
679  if (num_points == 0) {
680  vtkfile << "# vtk DataFile Version 3.0\n";
681  vtkfile << "Hurricane Track\n";
682  vtkfile << "ASCII\n";
683  vtkfile << "DATASET POLYDATA\n";
684  vtkfile << "POINTS " << num_points << " float\n";
685  vtkfile.close();
686  return;
687  }
688  if (num_points < 2) {
689  points_xy.push_back(points_xy[0]);
690  }
691  num_points = points_xy.size();
692 
693  vtkfile << "# vtk DataFile Version 3.0\n";
694  vtkfile << "Hurricane Track\n";
695  vtkfile << "ASCII\n";
696  vtkfile << "DATASET POLYDATA\n";
697 
698  // Write points (Z=0 assumed)
699  vtkfile << "POINTS " << num_points << " float\n";
700  for (const auto& pt : points_xy) {
701  vtkfile << pt[0] << " " << pt[1] << " 10000.0\n";
702  }
703 
704  // Write polyline connectivity
705  vtkfile << "LINES 1 " << num_points + 1 << "\n";
706  vtkfile << num_points << " ";
707  for (int i = 0; i < num_points; ++i) {
708  vtkfile << i << " ";
709  }
710  vtkfile << "\n";
711 
712  vtkfile.close();
713 }

Member Data Documentation

◆ advflux_reg

amrex::Vector<amrex::YAFluxRegister*> ERF::advflux_reg
private

Referenced by getAdvFluxReg().

◆ avg_xmom

amrex::Vector<amrex::MultiFab> ERF::avg_xmom
private

◆ avg_ymom

amrex::Vector<amrex::MultiFab> ERF::avg_ymom
private

◆ avg_zmom

amrex::Vector<amrex::MultiFab> ERF::avg_zmom
private

◆ ax

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax
private

◆ ax_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax_src
private

◆ ay

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay
private

◆ ay_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay_src
private

◆ az

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az
private

◆ az_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az_src
private

◆ ba1d

amrex::Vector<amrex::BoxArray> ERF::ba1d
private

◆ ba2d

amrex::Vector<amrex::BoxArray> ERF::ba2d
private

◆ base_state

amrex::Vector<amrex::MultiFab> ERF::base_state
private

◆ base_state_new

amrex::Vector<amrex::MultiFab> ERF::base_state_new
private

◆ bndry_output_planes_interval

int ERF::bndry_output_planes_interval = -1
staticprivate

◆ bndry_output_planes_per

Real ERF::bndry_output_planes_per = -1.0
staticprivate

◆ bndry_output_planes_start_time

Real ERF::bndry_output_planes_start_time = 0.0
staticprivate

◆ boxes_at_level

amrex::Vector<amrex::Vector<amrex::Box> > ERF::boxes_at_level
private

◆ cf_set_width

int ERF::cf_set_width {0}
private

◆ cf_width

int ERF::cf_width {0}
private

◆ cfl

Real ERF::cfl = 0.8
staticprivate

◆ change_max

Real ERF::change_max = 1.1
staticprivate

◆ check_file

std::string ERF::check_file {"chk"}
private

◆ check_for_nans

int ERF::check_for_nans = 0
staticprivate

◆ column_file_name

std::string ERF::column_file_name = "column_data.nc"
staticprivate

◆ column_interval

int ERF::column_interval = -1
staticprivate

◆ column_loc_x

Real ERF::column_loc_x = 0.0
staticprivate

◆ column_loc_y

Real ERF::column_loc_y = 0.0
staticprivate

◆ column_per

Real ERF::column_per = -1.0
staticprivate

◆ cons_names

const amrex::Vector<std::string> ERF::cons_names
private
Initial value:
{"density", "rhotheta", "rhoKE", "rhoadv_0",
"rhoQ1", "rhoQ2", "rhoQ3",
"rhoQ4", "rhoQ5", "rhoQ6"}

◆ cosPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::cosPhi_m
private

◆ d_havg_density

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_density
private

◆ d_havg_pressure

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_pressure
private

◆ d_havg_qc

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qc
private

◆ d_havg_qv

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qv
private

◆ d_havg_temperature

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_temperature
private

◆ d_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_rayleigh_ptrs
private

◆ d_rhoqt_src

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_rhoqt_src
private

◆ d_rhotheta_src

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_rhotheta_src
private

◆ d_sinesq_ptrs

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_sinesq_ptrs
private

◆ d_sinesq_stag_ptrs

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_sinesq_stag_ptrs
private

◆ d_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_sponge_ptrs
private

◆ d_u_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_u_geos
private

◆ d_v_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_v_geos
private

◆ d_w_subsid

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_w_subsid
private

◆ datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::datalog
private

◆ datalogname

amrex::Vector<std::string> ERF::datalogname
private

Referenced by DataLogName().

◆ datetime_format

const std::string ERF::datetime_format = "%Y-%m-%d %H:%M:%S"
private

◆ datprecision

const int ERF::datprecision = 6
private

◆ datwidth

const int ERF::datwidth = 14
private

◆ der_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::der_datalog
private

◆ der_datalogname

amrex::Vector<std::string> ERF::der_datalogname
private

Referenced by DerDataLogName().

◆ derived_names

const amrex::Vector<std::string> ERF::derived_names
private

◆ derived_names_2d

const amrex::Vector<std::string> ERF::derived_names_2d
private
Initial value:
{
"z_surf", "landmask", "mapfac", "lat_m", "lon_m",
"u_star", "w_star", "t_star", "q_star", "Olen", "pblh",
"t_surf", "q_surf", "z0", "OLR", "sens_flux", "laten_flux",
"surf_pres"
}

◆ derived_subvol_names

const amrex::Vector<std::string> ERF::derived_subvol_names {"soundspeed", "temp", "theta", "KE", "scalar"}
private

◆ destag_profiles

bool ERF::destag_profiles = true
private

◆ detJ_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc
private

◆ detJ_cc_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_new
private

◆ detJ_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_src
private

◆ domain_bc_type

amrex::Array<std::string,2*AMREX_SPACEDIM> ERF::domain_bc_type
private

◆ domain_bcs_type

amrex::Vector<amrex::BCRec> ERF::domain_bcs_type
private

◆ domain_bcs_type_d

amrex::Gpu::DeviceVector<amrex::BCRec> ERF::domain_bcs_type_d
private

◆ dt

amrex::Vector<amrex::Real> ERF::dt
private

◆ dt_max

Real ERF::dt_max = 1.0e9
staticprivate

◆ dt_max_initial

Real ERF::dt_max_initial = 2.0e100
staticprivate

◆ dt_mri_ratio

amrex::Vector<long> ERF::dt_mri_ratio
private

◆ dz_min

amrex::Vector<amrex::Real> ERF::dz_min
private

◆ eb

amrex::Vector<std::unique_ptr<eb_> > ERF::eb
private

Referenced by EBFactory(), and get_eb().

◆ eddyDiffs_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::eddyDiffs_lev
private

◆ fine_mask

amrex::MultiFab ERF::fine_mask
private

◆ finished_wave

bool ERF::finished_wave = false
private

◆ fixed_dt

amrex::Vector<amrex::Real> ERF::fixed_dt
private

◆ fixed_fast_dt

amrex::Vector<amrex::Real> ERF::fixed_fast_dt
private

◆ fixed_mri_dt_ratio

int ERF::fixed_mri_dt_ratio = 0
staticprivate

◆ forecast_state_1

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_1

◆ forecast_state_2

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_2

◆ forecast_state_interp

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_interp

◆ FPr_c

amrex::Vector<ERFFillPatcher> ERF::FPr_c
private

◆ FPr_u

amrex::Vector<ERFFillPatcher> ERF::FPr_u
private

◆ FPr_v

amrex::Vector<ERFFillPatcher> ERF::FPr_v
private

◆ FPr_w

amrex::Vector<ERFFillPatcher> ERF::FPr_w
private

◆ gradp

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::gradp
private

◆ h_havg_density

amrex::Vector<amrex::Real> ERF::h_havg_density
private

◆ h_havg_pressure

amrex::Vector<amrex::Real> ERF::h_havg_pressure
private

◆ h_havg_qc

amrex::Vector<amrex::Real> ERF::h_havg_qc
private

◆ h_havg_qv

amrex::Vector<amrex::Real> ERF::h_havg_qv
private

◆ h_havg_temperature

amrex::Vector<amrex::Real> ERF::h_havg_temperature
private

◆ h_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Vector<amrex::Real> > > ERF::h_rayleigh_ptrs
private

◆ h_rhoqt_src

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_rhoqt_src
private

◆ h_rhotheta_src

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_rhotheta_src
private

◆ h_sinesq_ptrs

amrex::Vector<amrex::Vector<amrex::Real> > ERF::h_sinesq_ptrs
private

◆ h_sinesq_stag_ptrs

amrex::Vector<amrex::Vector<amrex::Real> > ERF::h_sinesq_stag_ptrs
private

◆ h_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Vector<amrex::Real> > > ERF::h_sponge_ptrs
private

◆ h_u_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_u_geos
private

◆ h_v_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_v_geos
private

◆ h_w_subsid

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_w_subsid
private

◆ hurricane_eye_track_latlon

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_eye_track_latlon

◆ hurricane_eye_track_xy

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_eye_track_xy

◆ hurricane_maxvel_vs_time

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_maxvel_vs_time

◆ hurricane_track_xy

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_track_xy

◆ hurricane_tracker_circle

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_tracker_circle

◆ Hwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave
private

◆ Hwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave_onegrid
private

◆ init_shrink

Real ERF::init_shrink = 1.0
staticprivate

◆ input_bndry_planes

int ERF::input_bndry_planes = 0
staticprivate

◆ input_sounding_data

InputSoundingData ERF::input_sounding_data
private

◆ input_sponge_data

InputSpongeData ERF::input_sponge_data
private

◆ interpolation_type

StateInterpType ERF::interpolation_type
staticprivate

◆ istep

amrex::Vector<int> ERF::istep
private

◆ lagged_delta_rt

amrex::Vector<amrex::MultiFab> ERF::lagged_delta_rt
private

◆ land_type_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::land_type_lev
private

◆ last_check_file_step

int ERF::last_check_file_step = -1
staticprivate

◆ last_check_file_time

Real ERF::last_check_file_time = 0.0
staticprivate

◆ last_plot2d_file_step_1

int ERF::last_plot2d_file_step_1 = -1
staticprivate

◆ last_plot2d_file_step_2

int ERF::last_plot2d_file_step_2 = -1
staticprivate

◆ last_plot2d_file_time_1

Real ERF::last_plot2d_file_time_1 = 0.0
staticprivate

◆ last_plot2d_file_time_2

Real ERF::last_plot2d_file_time_2 = 0.0
staticprivate

◆ last_plot3d_file_step_1

int ERF::last_plot3d_file_step_1 = -1
staticprivate

◆ last_plot3d_file_step_2

int ERF::last_plot3d_file_step_2 = -1
staticprivate

◆ last_plot3d_file_time_1

Real ERF::last_plot3d_file_time_1 = 0.0
staticprivate

◆ last_plot3d_file_time_2

Real ERF::last_plot3d_file_time_2 = 0.0
staticprivate

◆ last_subvol_step

int ERF::last_subvol_step = -1
staticprivate

◆ last_subvol_time

Real ERF::last_subvol_time = 0.0
staticprivate

◆ lat_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lat_m
private

◆ line_sampler

std::unique_ptr<LineSampler> ERF::line_sampler = nullptr
private

◆ line_sampling_interval

int ERF::line_sampling_interval = -1
private

◆ line_sampling_per

amrex::Real ERF::line_sampling_per = -1.0
private

◆ lmask_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::lmask_lev
private

◆ lon_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lon_m
private

◆ lsm

LandSurface ERF::lsm
private

◆ lsm_data

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_data
private

◆ lsm_data_name

amrex::Vector<std::string> ERF::lsm_data_name
private

◆ lsm_flux

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_flux
private

◆ lsm_flux_name

amrex::Vector<std::string> ERF::lsm_flux_name
private

◆ Lwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave
private

◆ Lwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave_onegrid
private

◆ m_bc_extdir_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_extdir_vals
private

◆ m_bc_neumann_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_neumann_vals
private

◆ m_check_int

int ERF::m_check_int = -1
private

◆ m_check_per

amrex::Real ERF::m_check_per = -1.0
private

◆ m_expand_plotvars_to_unif_rr

bool ERF::m_expand_plotvars_to_unif_rr = false
private

◆ m_forest_drag

amrex::Vector<std::unique_ptr<ForestDrag> > ERF::m_forest_drag
private

◆ m_plot2d_int_1

int ERF::m_plot2d_int_1 = -1
private

◆ m_plot2d_int_2

int ERF::m_plot2d_int_2 = -1
private

◆ m_plot2d_per_1

amrex::Real ERF::m_plot2d_per_1 = -1.0
private

◆ m_plot2d_per_2

amrex::Real ERF::m_plot2d_per_2 = -1.0
private

◆ m_plot3d_int_1

int ERF::m_plot3d_int_1 = -1
private

◆ m_plot3d_int_2

int ERF::m_plot3d_int_2 = -1
private

◆ m_plot3d_per_1

amrex::Real ERF::m_plot3d_per_1 = -1.0
private

◆ m_plot3d_per_2

amrex::Real ERF::m_plot3d_per_2 = -1.0
private

◆ m_plot_face_vels

bool ERF::m_plot_face_vels = false
private

◆ m_r2d

std::unique_ptr<ReadBndryPlanes> ERF::m_r2d = nullptr
private

◆ m_subvol_int

int ERF::m_subvol_int = -1
private

◆ m_subvol_per

amrex::Real ERF::m_subvol_per = -1.0
private

◆ m_SurfaceLayer

std::unique_ptr<SurfaceLayer> ERF::m_SurfaceLayer = nullptr
private

◆ m_w2d

std::unique_ptr<WriteBndryPlanes> ERF::m_w2d = nullptr
private

◆ mapfac

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::mapfac
private

◆ max_step

int ERF::max_step = -1
private

◆ metgrid_basic_linear

bool ERF::metgrid_basic_linear {false}
private

◆ metgrid_debug_dry

bool ERF::metgrid_debug_dry {false}
private

◆ metgrid_debug_isothermal

bool ERF::metgrid_debug_isothermal {false}
private

◆ metgrid_debug_msf

bool ERF::metgrid_debug_msf {false}
private

◆ metgrid_debug_psfc

bool ERF::metgrid_debug_psfc {false}
private

◆ metgrid_debug_quiescent

bool ERF::metgrid_debug_quiescent {false}
private

◆ metgrid_force_sfc_k

int ERF::metgrid_force_sfc_k {6}
private

◆ metgrid_interp_theta

bool ERF::metgrid_interp_theta {false}
private

◆ metgrid_order

int ERF::metgrid_order {2}
private

◆ metgrid_proximity

amrex::Real ERF::metgrid_proximity {500.0}
private

◆ metgrid_retain_sfc

bool ERF::metgrid_retain_sfc {false}
private

◆ metgrid_use_below_sfc

bool ERF::metgrid_use_below_sfc {true}
private

◆ metgrid_use_sfc

bool ERF::metgrid_use_sfc {true}
private

◆ mf_C1H

std::unique_ptr<amrex::MultiFab> ERF::mf_C1H
private

◆ mf_C2H

std::unique_ptr<amrex::MultiFab> ERF::mf_C2H
private

◆ mf_MUB

std::unique_ptr<amrex::MultiFab> ERF::mf_MUB
private

◆ mf_PSFC

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mf_PSFC
private

◆ mg_verbose

int ERF::mg_verbose = 0
staticprivate

◆ micro

std::unique_ptr<Microphysics> ERF::micro
private

◆ mri_integrator_mem

amrex::Vector<std::unique_ptr<MRISplitIntegrator<amrex::Vector<amrex::MultiFab> > > > ERF::mri_integrator_mem
private

◆ nc_bdy_file

std::string ERF::nc_bdy_file
staticprivate

◆ nc_init_file

Vector< Vector< std::string > > ERF::nc_init_file = {{""}}
staticprivate

◆ nc_low_file

std::string ERF::nc_low_file
staticprivate

◆ ng_dens_hse

int ERF::ng_dens_hse
staticprivate

◆ ng_pres_hse

int ERF::ng_pres_hse
staticprivate

◆ nsubsteps

amrex::Vector<int> ERF::nsubsteps
private

◆ num_boxes_at_level

amrex::Vector<int> ERF::num_boxes_at_level
private

◆ num_files_at_level

amrex::Vector<int> ERF::num_files_at_level
private

◆ output_1d_column

int ERF::output_1d_column = 0
staticprivate

◆ output_bndry_planes

int ERF::output_bndry_planes = 0
staticprivate

◆ pert_interval

int ERF::pert_interval = -1
staticprivate

◆ phys_bc_type

amrex::GpuArray<ERF_BC, AMREX_SPACEDIM*2> ERF::phys_bc_type
private

◆ physbcs_base

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_base> > ERF::physbcs_base
private

◆ physbcs_cons

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_cons> > ERF::physbcs_cons
private

◆ physbcs_u

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_u> > ERF::physbcs_u
private

◆ physbcs_v

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_v> > ERF::physbcs_v
private

◆ physbcs_w

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_w> > ERF::physbcs_w
private

◆ plane_sampler

std::unique_ptr<PlaneSampler> ERF::plane_sampler = nullptr
private

◆ plane_sampling_interval

int ERF::plane_sampling_interval = -1
private

◆ plane_sampling_per

amrex::Real ERF::plane_sampling_per = -1.0
private

◆ plot2d_file_1

std::string ERF::plot2d_file_1 {"plt2d_1_"}
private

◆ plot2d_file_2

std::string ERF::plot2d_file_2 {"plt2d_2_"}
private

◆ plot2d_var_names_1

amrex::Vector<std::string> ERF::plot2d_var_names_1
private

◆ plot2d_var_names_2

amrex::Vector<std::string> ERF::plot2d_var_names_2
private

◆ plot3d_file_1

std::string ERF::plot3d_file_1 {"plt_1_"}
private

◆ plot3d_file_2

std::string ERF::plot3d_file_2 {"plt_2_"}
private

◆ plot3d_var_names_1

amrex::Vector<std::string> ERF::plot3d_var_names_1
private

◆ plot3d_var_names_2

amrex::Vector<std::string> ERF::plot3d_var_names_2
private

◆ plot_file_on_restart

bool ERF::plot_file_on_restart = true
staticprivate

◆ plot_lsm

bool ERF::plot_lsm = false
private

◆ plot_rad

bool ERF::plot_rad = false
private

◆ plotfile2d_type_1

PlotFileType ERF::plotfile2d_type_1 = PlotFileType::None
staticprivate

◆ plotfile2d_type_2

PlotFileType ERF::plotfile2d_type_2 = PlotFileType::None
staticprivate

◆ plotfile3d_type_1

PlotFileType ERF::plotfile3d_type_1 = PlotFileType::None
staticprivate

◆ plotfile3d_type_2

PlotFileType ERF::plotfile3d_type_2 = PlotFileType::None
staticprivate

◆ pp_inc

amrex::Vector<amrex::MultiFab> ERF::pp_inc
private

◆ pp_prefix

std::string ERF::pp_prefix {"erf"}

◆ previousCPUTimeUsed

Real ERF::previousCPUTimeUsed = 0.0
staticprivate

Referenced by getCPUTime().

◆ prob

std::unique_ptr<ProblemBase> ERF::prob = nullptr
private

◆ profile_int

int ERF::profile_int = -1
private

◆ qheating_rates

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::qheating_rates
private

◆ qmoist

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::qmoist
private

◆ Qr_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qr_prim
private

◆ Qv_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qv_prim
private

◆ rad

amrex::Vector<std::unique_ptr<IRadiation> > ERF::rad
private

◆ rad_datalog_int

int ERF::rad_datalog_int = -1
private

◆ rad_fluxes

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::rad_fluxes
private

◆ real_extrap_w

bool ERF::real_extrap_w {true}
private

◆ real_set_width

int ERF::real_set_width {0}
private

◆ real_width

int ERF::real_width {0}
private

◆ ref_tags

Vector< AMRErrorTag > ERF::ref_tags
staticprivate

◆ regrid_int

int ERF::regrid_int = -1
private

◆ regrid_level_0_on_restart

bool ERF::regrid_level_0_on_restart = false
private

◆ restart_chkfile

std::string ERF::restart_chkfile = ""
private

◆ rU_new

amrex::Vector<amrex::MultiFab> ERF::rU_new
private

◆ rU_old

amrex::Vector<amrex::MultiFab> ERF::rU_old
private

◆ rV_new

amrex::Vector<amrex::MultiFab> ERF::rV_new
private

◆ rV_old

amrex::Vector<amrex::MultiFab> ERF::rV_old
private

◆ rW_new

amrex::Vector<amrex::MultiFab> ERF::rW_new
private

◆ rW_old

amrex::Vector<amrex::MultiFab> ERF::rW_old
private

◆ sampleline

amrex::Vector<amrex::IntVect> ERF::sampleline
private

Referenced by NumSampleLines(), and SampleLine().

◆ samplelinelog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::samplelinelog
private

◆ samplelinelogname

amrex::Vector<std::string> ERF::samplelinelogname
private

Referenced by SampleLineLogName().

◆ samplepoint

amrex::Vector<amrex::IntVect> ERF::samplepoint
private

Referenced by NumSamplePoints(), and SamplePoint().

◆ sampleptlog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::sampleptlog
private

◆ sampleptlogname

amrex::Vector<std::string> ERF::sampleptlogname
private

Referenced by SamplePointLogName().

◆ SFS_diss_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_diss_lev
private

◆ SFS_hfx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx1_lev
private

◆ SFS_hfx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx2_lev
private

◆ SFS_hfx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx3_lev
private

◆ SFS_q1fx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx1_lev
private

◆ SFS_q1fx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx2_lev
private

◆ SFS_q1fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx3_lev
private

◆ SFS_q2fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q2fx3_lev
private

◆ sinPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::sinPhi_m
private

◆ SmnSmn_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SmnSmn_lev
private

◆ soil_type_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::soil_type_lev
private

◆ solar_zenith

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::solar_zenith
private

◆ solverChoice

SolverChoice ERF::solverChoice
staticprivate

◆ sponge_type

std::string ERF::sponge_type
staticprivate

◆ sst_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::sst_lev
private

◆ start_time

Real ERF::start_time = 0.0
staticprivate

◆ startCPUTime

Real ERF::startCPUTime = 0.0
staticprivate

Referenced by getCPUTime().

◆ stop_time

Real ERF::stop_time = std::numeric_limits<amrex::Real>::max()
staticprivate

◆ stretched_dz_d

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::stretched_dz_d
private

◆ stretched_dz_h

amrex::Vector<amrex::Vector<amrex::Real> > ERF::stretched_dz_h
private

◆ sub_cfl

Real ERF::sub_cfl = 1.0
staticprivate

◆ subdomains

amrex::Vector<amrex::Vector<amrex::BoxArray> > ERF::subdomains
private

◆ subvol3d_var_names

amrex::Vector<std::string> ERF::subvol3d_var_names
private

◆ subvol_file

std::string ERF::subvol_file {"subvol"}
private

◆ sum_interval

int ERF::sum_interval = -1
staticprivate

◆ sum_per

Real ERF::sum_per = -1.0
staticprivate

◆ sw_lw_fluxes

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::sw_lw_fluxes
private

◆ t_avg_cnt

amrex::Vector<amrex::Real> ERF::t_avg_cnt
private

◆ t_new

amrex::Vector<amrex::Real> ERF::t_new
private

◆ t_old

amrex::Vector<amrex::Real> ERF::t_old
private

◆ Tau

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::Tau
private

◆ Tau_corr

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::Tau_corr
private

◆ terrain_blanking

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::terrain_blanking
private

◆ th_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::th_bc_data
private

◆ Theta_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Theta_prim
private

◆ thin_xforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_xforce
private

◆ thin_yforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_yforce
private

◆ thin_zforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_zforce
private

◆ timeprecision

const int ERF::timeprecision = 13
private

◆ tot_e_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::tot_e_datalog
private

Referenced by setRecordEnergyDataInfo().

◆ tot_e_datalogname

amrex::Vector<std::string> ERF::tot_e_datalogname
private

◆ tsk_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::tsk_lev
private

◆ turbPert

TurbulentPerturbation ERF::turbPert
private

◆ urb_frac_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::urb_frac_lev
private

◆ use_datetime

bool ERF::use_datetime = false
private

◆ use_fft

bool ERF::use_fft = false
staticprivate

◆ vars_new

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_new
private

◆ vars_old

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_old
private

◆ vel_t_avg

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::vel_t_avg
private

◆ verbose

int ERF::verbose = 0
staticprivate

◆ walldist

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::walldist
private

◆ weather_forecast_data_1

amrex::Vector<amrex::MultiFab> ERF::weather_forecast_data_1

◆ weather_forecast_data_2

amrex::Vector<amrex::MultiFab> ERF::weather_forecast_data_2

◆ xflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::xflux_imask
private

◆ xvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::xvel_bc_data
private

◆ yflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::yflux_imask
private

◆ yvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::yvel_bc_data
private

◆ z_phys_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc
private

◆ z_phys_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc_src
private

◆ z_phys_nd

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd
private

◆ z_phys_nd_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_new
private

◆ z_phys_nd_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_src
private

◆ z_t_rk

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_t_rk
private

◆ zflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::zflux_imask
private

◆ zlevels_stag

amrex::Vector<amrex::Vector<amrex::Real> > ERF::zlevels_stag
private

◆ zmom_crse_rhs

amrex::Vector<amrex::MultiFab> ERF::zmom_crse_rhs
private

◆ zvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::zvel_bc_data
private

The documentation for this class was generated from the following files: